#!/usr/bin/perl
# ---------------------------------------------------------------------------------------------
#
# ck_corpus.pl --- checks popfile corpus for imbedded spaces and non
#                  integer word counts
#
# This program authored by Scott W Leighton (helphand@pacbell.net)
# as a utility program for Popfile, which is Copyrighted
# by John Graham-Cumming. 
#
#    ck_corpus.pl
#    Copyright (C) 2003  Scott W. Leighton helphand@pacbell.net
#
#    Modified May 24, 2003 - Make corpus table optional
#
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#
#
# Popfile
# Copyright (c) 2001-2003 John Graham-Cumming
#
# ---------------------------------------------------------------------------------------------


use strict;

    my @buckets = glob ("corpus/*");

    foreach my $bucket (@buckets) {
        my $line = 0;
        print "checking bucket $bucket\n";
        if (open FILE, "<${bucket}/table" ) {
           while (<FILE>) {
               if ( /__CORPUS__ __VERSION__ (\d+)/ ) {
                   next;
               }
               $line++;
               my @w = split /\s/,$_;
               if (scalar( $#w) > 1 ) {
                   print "Bucket $bucket line $line has a problem '$w[0] $w[1] $w[2]'\n";
               } else {
                   $w[1] =~ s/\n|\r//g;
                   unless ( $w[1] =~ /^\d{1,9}$/ ) {
                       print "Bucket $bucket line $line has missing or non integer word count '$w[0] ($w[1])'\n";
                   }
               }
           }

           close FILE;
        } else {
           print "Warning, unable to open bucket $bucket table file\n";
        }

    }