#This file is in the public domain
#Originally written by proteos for its own use (epi.proteos.info)
#Usage: perl <name_of_this_file>
#You must have perl & the various packages used (after the keyword 'use') installed
#You should also modify the start date and end date to get different data sets
#Outputs in a file named "fr_prices.tsv"
#It is a file where values are tab separated
#This script relies on a version of epexspot's website in service in 2013
#It may cease to work when epexspot revamps its website
#Although I can't see how you may take any risk with this file, 
#use at your own risk.

use DateTime;
use URI;
use LWP::UserAgent;
use HTML::TreeBuilder;
use File::Fetch;

my $startDate = new DateTime( 
	year => 2013, 
	month => 1, 
	day => 1, 
	time_zone => 'Europe/Paris'
	);
	
my $endDate = new DateTime(
	year	=>	2013,
	month	=>	4,
	day		=>	1,
	time_zone => 'Europe/Paris'
	);

my $cDate = $startDate->clone(); #current date
my $ua = LWP::UserAgent->new();

open (my $fo, ">fr_prices.tsv");

while ( $cDate <= $endDate )
{
	my $dstr = $cDate->ymd;
	my $urstr = "https://www.epexspot.com/en/market-data/auction/auction-table/" . $dstr . "/FR";
	
	my $hres = $ua->get($urstr);
	my $tree = HTML::TreeBuilder->new;
	$tree->parse($hres->decoded_content);
	$tree->eof();
	my @tables = $tree->look_down('_tag','table','class','list hours');
	# on se retrouve avec 3 tables
	# la premiere est pour la france
	# la deuxieme pour allemagne + autriche
	# la troisieme pour la suisse
	#print "tables: $#tables \n";
	
	my $fr_table = $tables[0];
	my $de_table = $tables[1];
	my @rows = $fr_table->look_down('_tag','tr');
	
	#print $fh "$#tables \n";
	#print $fh "$#rows \n";
	
	my @fvector;
	my $nh = int( ($#rows-1)/2 );

	foreach my $row (0 .. $#rows)
	{
		my $hd = int($row/2); #hour of the day
		if ( $row % 2 == 1 ) 
		{ #indication de la tranche horaire
			#print $fo $rows[$row]->as_text . "\n";
			my @elts = $rows[$row]->look_down('_tag','td');
			foreach my $elt (2 .. $#elts)
			{
				my $dw = $elt - 2; #day of week
				#print $fo $elts[$elt]->as_text . "\t";
				
				if ( ($elts[$elt]->as_text) =~ /\d/ )
				{
					$fvector[$dw*$nh+$hd] = $elts[$elt]->as_text;
					#$pindex++;
				}
				else 
				{
					$fvector[$dw*$#rows+$hd] = undef;
				}
				#print $fh $elt->as_text . "\t";
			}
			#print $fo "\n";	
		}
	
	}
	
	my @dvector;
	@rows = $de_table->look_down('_tag','tr');
	$nh = int( ($#rows-1)/2 );
	foreach my $row (0 .. $#rows)
	{
		my $hd = int($row/2); #hour of the day
		if ( $row % 2 == 1 ) 
		{ #indication de la tranche horaire
			#print $fo $rows[$row]->as_text . "\n";
			my @elts = $rows[$row]->look_down('_tag','td');
			foreach my $elt (2 .. $#elts)
			{
				my $dw = $elt - 2; #day of week
				#print $fo $elts[$elt]->as_text . "\t";
				
				if ( ($elts[$elt]->as_text) =~ /\d/ )
				{
					$dvector[$dw*$nh+$hd] = $elts[$elt]->as_text;
					#$pindex++;
				}
				else 
				{
					$dvector[$dw*$#rows+$hd] = undef;
				}
				#print $fh $elt->as_text . "\t";
			}
			#print $fo "\n";	
		}
	
	}
	
	my $wDate = $cDate->clone;
	
	foreach my $d (0 .. 6)
	{
		#while the number of hours in a day changes, 
		#the number of days in week does not
		foreach my $h (0 .. $nh )
		{
			if ( $fvector[$d*$nh+$h] != undef )
			{
				my $dstr = $wDate->ymd . "\t" . $wDate->hour; 
				print $fo $dstr. "\t". $fvector[$d*$nh+$h] . "\t". $dvector[$d*$nh+$h] . "\n";
				$wDate->add( hours => 1 );
			}
		}
		
	}
	
	$tree->delete;
	$cDate->add( days => 7 );
	sleep(1);
		
} 


#print $fh "stop\n";
close($fo);