use strict ; use warnings ; use Data::Dumper ; use Spreadsheet::XLSX; use DateTime::Format::Excel ; use File::Slurp ; use Safelib::Dbutils ; my @tab_descs ; my $dbh ; my $excel_file = shift @ARGV or die 'missing excel file argument' ; my $tab_info = preprocess_tab_descs() ; #print Dumper $tab_info ; #exit ; my $tab_rows = get_tab_data( $tab_info, $excel_file ) ; #print Dumper $tab_rows ; output_tab_files( $tab_info, $tab_rows ) ; exit ; sub get_tab_data { my( $tab_info, $excel_file ) = @_ ; my $excel = Spreadsheet::XLSX->new( $excel_file ); my $cells = $excel->{Worksheet}[0]{Cells} ; my $tab_xl_num = $tab_info->{tab_xl_num} ; #print Dumper $cells ; #exit ; # skip the first excel row which is just column names shift @{$cells} ; my @tab_rows ; foreach my $row ( @{$cells} ) { my %tab_row ; while( my( $tab_name, $xl_col ) = each %{$tab_xl_num} ) { $tab_row{$tab_name} = $row->[$xl_col]{Val} ; } # get the pif data and add it to the tab row data get_pif_data( $tab_info, \%tab_row, ) ; push @tab_rows, \%tab_row ; } #print Dumper \@tab_rows ; return \@tab_rows ; } sub get_pif_data { my( $tab_info, $tab_row ) = @_ ; my $pif_sql = $tab_info->{pif_sql} ; $dbh ||= Safelib::Dbutils->connect( $ENV{TAB_DB} || 'LocalServer' ) or die "Connect failed: $DBI::errstr" ; my $master_id = get_pif_master_id( $tab_row ) ; while( my( $pif_rec, $sql ) = each %{$pif_sql} ) { # get the rows from this pif table my $pif_recs = $dbh->exec_select( $sql, $master_id ) ; # we only use the first row (can we have multiple rows from the db?) my $rec = $pif_recs->[0] ; # copy all the values to the @{$tab_row}{ keys %{$rec} } = values %{$rec} ; } } sub get_pif_master_id { my( $tab_row ) = @_ ; my $excel_date = $tab_row->{POLICY_EFFECTIVE_DATE} ; my $datetime = DateTime::Format::Excel->parse_datetime( $excel_date ); my $eff_date = $datetime->ymd('') ; my $policy_num = "$tab_row->{POLICY_SYMBOL}$tab_row->{POLICY_NUMBER}" ; my $sql = <exec_select( $sql, $policy_num, $eff_date ) ; return $master_row->[0]{id} ; } sub output_tab_files { my( $tab_info, $tab_rows ) = @_ ; my @tab_names = @{$tab_info->{tab_names}} ; my $defaults = $tab_info->{defaults} ; my $tab_header = join( "\t", map qq{"$_"}, @tab_names ) ; my %tab_texts ; foreach my $tab_row ( @{$tab_rows} ) { foreach my $tab_name ( @tab_names ) { # set any defaults or null string and strip surrounding blanks $tab_row->{$tab_name} //= $defaults->{$tab_name} ; s/^\s+//, s/\s+$// for $tab_row->{$tab_name} ; } my $tab_line = join "\t", map qq{"$_"}, @{$tab_row}{@tab_names} ; my $cat_firm = $tab_row->{CAT_FIRM} ; # put in a tab column header line if we haven't seen this cat firm before $tab_texts{$cat_firm} ||= "$tab_header\n" ; # add in the data row $tab_texts{$cat_firm} .= "$tab_line\n" ; } while( my( $cat_firm, $tab_text ) = each %tab_texts ) { $cat_firm =~ tr/ /_/ ; write_file( "$cat_firm.tsv", $tab_text ) ; } } sub preprocess_tab_descs { # these will all get stuffed with munged data from the field specs my @tab_names ; # list of tab field names my %defaults ; # default value for each field my %tab_xl_num ; # field name to excel column number my %tab_pif_map ; # field name to pif field name # map of default types to values. only needed one time so it can # reside in this sub. my %default_vals = ( date => '0001-01-01', zero => '0', ) ; # loop over all the tab descriptions to process and sort them. foreach my $desc ( @tab_descs ) { # let's look at this tab field descriptor my( $tab_name, $rec_type, $col, $default_type ) = @{$desc} ; # we keep all the names in sequence for printing the tab row push( @tab_names, $tab_name ) ; # get the default value for this field or it is the null string $defaults{ $tab_name } = $default_type ? $default_vals{$default_type} : '' ; # nothing more to do if this is just a placeholder (to be filled in manually) next unless $rec_type ; # if an excel field, save the column number if ( $rec_type eq 'XL' ) { $tab_xl_num{ $tab_name } = $col ; next ; } # must be a PIF field, save this field desc with others for this record type push( @{$tab_pif_map{$rec_type}}, $desc ) ; } my $pif_sql = generate_pif_sql( \%tab_pif_map ) ; return { tab_names => \@tab_names, tab_xl_num => \%tab_xl_num, pif_sql => $pif_sql, defaults => \%defaults, } ; } sub generate_pif_sql { my( $tab_pif_map ) = @_ ; my %pif_sql ; while( my( $pif_rec, $maps ) = each %{$tab_pif_map} ) { my $select = join ",\n\t", map "$_->[2] as $_->[0]", @{$maps} ; my $sql = <