compdic.c

Go to the documentation of this file.
00001 /* Eliot                                                                     */
00002 /* Copyright (C) 1999  Antoine Fraboulet                                     */
00003 /*                                                                           */
00004 /* This file is part of Eliot.                                               */
00005 /*                                                                           */
00006 /* Eliot is free software; you can redistribute it and/or modify             */
00007 /* it under the terms of the GNU General Public License as published by      */
00008 /* the Free Software Foundation; either version 2 of the License, or         */
00009 /* (at your option) any later version.                                       */
00010 /*                                                                           */
00011 /* Eliot is distributed in the hope that it will be useful,                  */
00012 /* but WITHOUT ANY WARRANTY; without even the implied warranty of            */
00013 /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             */
00014 /* GNU General Public License for more details.                              */
00015 /*                                                                           */
00016 /* You should have received a copy of the GNU General Public License         */
00017 /* along with this program; if not, write to the Free Software               */
00018 /* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA */
00019 
00020 /**
00021  *  \file   compdic.c
00022  *  \brief  Program used to compress a dictionary
00023  *  \author Antoine Fraboulet
00024  *  \date   1999
00025  */
00026 
00027 #include <time.h>
00028 #include <sys/types.h>
00029 #include <sys/stat.h>
00030 #include <fcntl.h>
00031 #include <stdlib.h>
00032 #include <stdio.h>
00033 #include <string.h>
00034 #include <ctype.h>
00035 #include <assert.h>
00036 #include "hashtable.h"
00037 #include "dic_internals.h"
00038 
00039 //#define DEBUG_LIST
00040 //#define DEBUG_OUTPUT
00041 //#define DEBUG_OUTPUT_L2
00042 #define CHECK_RECURSION
00043 
00044 char*
00045 load_uncompressed(const char* file_name, unsigned int *dic_size)
00046 {
00047   unsigned r;
00048   char *uncompressed;
00049   FILE* file_desc;
00050 
00051   if ((file_desc = fopen (file_name, "r")) == NULL)
00052     return NULL;
00053 
00054   if ((uncompressed = (char*)malloc (sizeof(char)*(*dic_size))) == NULL)
00055     return NULL;
00056 
00057   r = fread (uncompressed, 1, *dic_size, file_desc);
00058   if (r < *dic_size)
00059     {
00060       /* \n is 2 chars under MS OS */
00061       printf("\n");
00062       printf("** The number of bytes read is less than the size of the file **\n");
00063       printf("** this may be OK if you run a Microsoft OS but not on Unix   **\n");
00064       printf("** please check the results.                                  **\n");
00065       printf("\n");
00066       *dic_size = r;
00067     }
00068 
00069   fclose(file_desc);
00070   return uncompressed;
00071 }
00072 
00073 
00074 int
00075 file_length(const char* file_name)
00076 {
00077   struct stat stat_buf;
00078   if (stat (file_name, &stat_buf) < 0)
00079     return - 1;
00080   return (int) stat_buf.st_size;
00081 }
00082 
00083 
00084 void
00085 skip_init_header(FILE* outfile, Dict_header *header)
00086 {
00087   header->unused_1   = 0;
00088   header->unused_2   = 0;
00089   header->root       = 0;
00090   header->nwords     = 0;
00091   header->nodesused  = 1;
00092   header->edgesused  = 1;
00093   header->nodessaved = 0;
00094   header->edgessaved = 0;
00095 
00096   fwrite (header, sizeof(Dict_header), 1, outfile);
00097 }
00098 
00099 
00100 void
00101 fix_header(FILE* outfile, Dict_header* header)
00102 {
00103   strcpy(header->ident,_COMPIL_KEYWORD_);
00104   header->root = header->edgesused;
00105   rewind (outfile);
00106   fwrite (header, sizeof(Dict_header), 1, outfile);
00107 }
00108 
00109 
00110 void
00111 print_header_info(Dict_header *header)
00112 {
00113   printf("============================\n");
00114   printf("keyword length %d bytes\n",(int)strlen(_COMPIL_KEYWORD_));
00115   printf("keyword size   %d bytes\n",sizeof(_COMPIL_KEYWORD_));
00116   printf("header size    %d bytes\n",sizeof(Dict_header));
00117   printf("\n");
00118   printf("%d words\n",header->nwords);
00119   printf("\n");
00120   printf("root : %7d (edge)\n",header->root);
00121   printf("root : %7d (byte)\n",header->root * sizeof(Dawg_edge));
00122   printf("\n");
00123   printf("nodes : %d+%d\n",header->nodesused, header->nodessaved);
00124   printf("edges : %d+%d\n",header->edgesused, header->edgessaved);
00125   printf("============================\n");
00126 }
00127 
00128 
00129 void
00130 write_node(Dawg_edge *edges, int size, int num, FILE* outfile)
00131 {
00132 #ifdef DEBUG_OUTPUT
00133   int i;
00134   printf("writing %d edges\n",num);
00135   for(i=0; i<num; i++)
00136     {
00137 #ifdef DEBUG_OUTPUT_L2
00138       printf("ptr=%2d t=%d l=%d f=%d chr=%2d (%c)\n",
00139              edges[i].ptr, edges[i].term, edges[i].last,
00140              edges[i].fill, edges[i].chr, edges[i].chr -1 +'a');
00141 #endif
00142       fwrite (edges+i, sizeof(Dawg_edge), 1, outfile);
00143     }
00144 #else
00145   fwrite (edges, size, num, outfile);
00146 #endif
00147 }
00148 
00149 #define MAX_STRING_LENGTH 200
00150 
00151 
00152 #define MAX_EDGES 2000
00153 /* ods3: ??   */
00154 /* ods4: 1746 */
00155 
00156 /* global variables */
00157 FILE*       global_outfile;
00158 Dict_header global_header;
00159 Hash_table  global_hashtable;
00160 
00161 char        global_stringbuf[MAX_STRING_LENGTH]; /* Space for current string */
00162 char*       global_endstring;                    /* Marks END of current string */
00163 char*       global_input;
00164 char*       global_endofinput;
00165 
00166 /*
00167  * Makenode takes a prefix (as position relative to stringbuf) and
00168  * returns an index of the start node of a dawg that recognizes all
00169  * words beginning with that prefix.  String is a pointer (relative
00170  * to stringbuf) indicating how much of prefix is matched in the
00171  * input.
00172  */
00173 #ifdef CHECK_RECURSION
00174 int current_rec =0;
00175 int max_rec = 0;
00176 #endif
00177 
00178 unsigned int
00179 makenode(char *prefix)
00180 {
00181   int    numedges;
00182   Dawg_edge  edges[MAX_EDGES];
00183   Dawg_edge *edgeptr = edges;
00184   unsigned  *saved_position;
00185 
00186 #ifdef CHECK_RECURSION
00187   current_rec++;
00188   if (current_rec > max_rec)
00189     max_rec = current_rec;
00190 #endif
00191 
00192   while (prefix == global_endstring)
00193     {
00194       /* More edges out of node */
00195       edgeptr->ptr  = 0;
00196       edgeptr->term = 0;
00197       edgeptr->last = 0;
00198       edgeptr->fill = 0;
00199       edgeptr->chr  = 0;
00200 
00201       (*(edgeptr++)).chr = (*global_endstring++ = *global_input++) & DIC_CHAR_MASK;
00202       if (*global_input == '\n')                 /* End of a word */
00203         {
00204           global_header.nwords++;
00205           edgeptr[-1].term = 1;                  /* Mark edge as word */
00206           *global_endstring++ = *global_input++; /* Skip \n */
00207           if (global_input == global_endofinput) /* At end of input? */
00208             break;
00209 
00210           global_endstring = global_stringbuf;
00211           while(*global_endstring == *global_input)
00212             {
00213                global_endstring++;
00214                global_input++;
00215             }
00216         }
00217       /* make dawg pointed to by this edge */
00218       edgeptr[-1].ptr = makenode(prefix + 1);
00219     }
00220 
00221   numedges = edgeptr - edges;
00222   if (numedges == 0)
00223     {
00224 #ifdef CHECK_RECURSION
00225       current_rec --;
00226 #endif
00227       return 0;             /* Special node zero - no edges */
00228     }
00229 
00230   edgeptr[-1].last = 1;     /* Mark the last edge */
00231 
00232   saved_position = (unsigned int*) hash_find (global_hashtable,
00233                                               (void*)edges,
00234                                               numedges*sizeof(Dawg_edge));
00235   if (saved_position)
00236     {
00237       global_header.edgessaved += numedges;
00238       global_header.nodessaved++;
00239 
00240 #ifdef CHECK_RECURSION
00241       current_rec --;
00242 #endif
00243       return *saved_position;
00244     }
00245   else
00246     {
00247       unsigned int node_pos;
00248 
00249       node_pos = global_header.edgesused;
00250       hash_add(global_hashtable,
00251                (void*)edges,numedges*sizeof(Dawg_edge),
00252                (void*)(&global_header.edgesused),sizeof(global_header.edgesused));
00253       global_header.edgesused += numedges;
00254       global_header.nodesused++;
00255       write_node (edges, sizeof(Dawg_edge), numedges, global_outfile);
00256 
00257 #ifdef CHECK_RECURSION
00258       current_rec --;
00259 #endif
00260       return node_pos;
00261     }
00262 }
00263 
00264 
00265 
00266 
00267 int
00268 main(int argc, char* argv[])
00269 {
00270   unsigned int dicsize;
00271   char *uncompressed;
00272   Dawg_edge rootnode = {0,0,0,0,0};
00273   Dawg_edge specialnode = {0,0,0,0,0};
00274 
00275   char* outfilename;
00276   char outfilenamedefault[] = "dict.daw";
00277   clock_t starttime, endtime;
00278 
00279   if (argc < 2)
00280     {
00281       fprintf(stderr,"usage: %s uncompressed_dic [compressed_dic]\n",argv[0]);
00282       exit(1);
00283     }
00284 
00285   dicsize = file_length (argv[1]);
00286   if (dicsize < 0)
00287     {
00288       fprintf(stderr,"Cannot stat uncompressed dictionary %s\n",argv[1]);
00289       exit(1);
00290     }
00291 
00292   outfilename = (argc == 3) ? argv[2] : outfilenamedefault;
00293 
00294   if ((global_outfile = fopen (outfilename,"wb")) == NULL)
00295     {
00296       fprintf(stderr,"Cannot open output file %s\n",outfilename);
00297       exit(1);
00298     }
00299 
00300   if ((uncompressed = load_uncompressed(argv[1], &dicsize)) == NULL)
00301     {
00302       fprintf(stderr,"Cannot load uncompressed dictionary into memory\n");
00303       exit(1);
00304     }
00305 
00306   global_input = uncompressed;
00307   global_endofinput = global_input + dicsize;
00308 
00309 #define SCALE 0.6
00310   global_hashtable = hash_init((unsigned int)(dicsize * SCALE));
00311 #undef SCALE
00312 
00313   skip_init_header(global_outfile,&global_header);
00314 
00315   specialnode.last = 1;
00316   write_node(&specialnode,sizeof(specialnode),1,global_outfile);
00317   /*
00318    * Call makenode with null (relative to stringbuf) prefix;
00319    * Initialize string to null; Put index of start node on output
00320    */
00321   starttime=clock();
00322   rootnode.ptr = makenode(global_endstring = global_stringbuf);
00323   endtime=clock();
00324   write_node(&rootnode,sizeof(rootnode),1,global_outfile);
00325 
00326   fix_header(global_outfile,&global_header);
00327 
00328   print_header_info(&global_header);
00329   hash_destroy(global_hashtable);
00330   free(uncompressed);
00331   fclose(global_outfile);
00332 
00333   printf(" Elapsed time is                 : %f s\n", 1.0*(endtime-starttime) / CLOCKS_PER_SEC);
00334 #ifdef CHECK_RECURSION
00335   printf(" Maximum recursion level reached : %d\n",max_rec);
00336 #endif
00337   return 0;
00338 }
00339 
00340 

Generated on Thu Dec 29 02:01:14 2005 for Eliot by  doxygen 1.4.5