00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027 #include <time.h>
00028 #include <sys/types.h>
00029 #include <sys/stat.h>
00030 #include <fcntl.h>
00031 #include <stdlib.h>
00032 #include <stdio.h>
00033 #include <string.h>
00034 #include <ctype.h>
00035 #include <assert.h>
00036 #include "hashtable.h"
00037 #include "dic_internals.h"
00038
00039
00040
00041
00042 #define CHECK_RECURSION
00043
00044 char*
00045 load_uncompressed(const char* file_name, unsigned int *dic_size)
00046 {
00047 unsigned r;
00048 char *uncompressed;
00049 FILE* file_desc;
00050
00051 if ((file_desc = fopen (file_name, "r")) == NULL)
00052 return NULL;
00053
00054 if ((uncompressed = (char*)malloc (sizeof(char)*(*dic_size))) == NULL)
00055 return NULL;
00056
00057 r = fread (uncompressed, 1, *dic_size, file_desc);
00058 if (r < *dic_size)
00059 {
00060
00061 printf("\n");
00062 printf("** The number of bytes read is less than the size of the file **\n");
00063 printf("** this may be OK if you run a Microsoft OS but not on Unix **\n");
00064 printf("** please check the results. **\n");
00065 printf("\n");
00066 *dic_size = r;
00067 }
00068
00069 fclose(file_desc);
00070 return uncompressed;
00071 }
00072
00073
00074 int
00075 file_length(const char* file_name)
00076 {
00077 struct stat stat_buf;
00078 if (stat (file_name, &stat_buf) < 0)
00079 return - 1;
00080 return (int) stat_buf.st_size;
00081 }
00082
00083
00084 void
00085 skip_init_header(FILE* outfile, Dict_header *header)
00086 {
00087 header->unused_1 = 0;
00088 header->unused_2 = 0;
00089 header->root = 0;
00090 header->nwords = 0;
00091 header->nodesused = 1;
00092 header->edgesused = 1;
00093 header->nodessaved = 0;
00094 header->edgessaved = 0;
00095
00096 fwrite (header, sizeof(Dict_header), 1, outfile);
00097 }
00098
00099
00100 void
00101 fix_header(FILE* outfile, Dict_header* header)
00102 {
00103 strcpy(header->ident,_COMPIL_KEYWORD_);
00104 header->root = header->edgesused;
00105 rewind (outfile);
00106 fwrite (header, sizeof(Dict_header), 1, outfile);
00107 }
00108
00109
00110 void
00111 print_header_info(Dict_header *header)
00112 {
00113 printf("============================\n");
00114 printf("keyword length %d bytes\n",(int)strlen(_COMPIL_KEYWORD_));
00115 printf("keyword size %d bytes\n",sizeof(_COMPIL_KEYWORD_));
00116 printf("header size %d bytes\n",sizeof(Dict_header));
00117 printf("\n");
00118 printf("%d words\n",header->nwords);
00119 printf("\n");
00120 printf("root : %7d (edge)\n",header->root);
00121 printf("root : %7d (byte)\n",header->root * sizeof(Dawg_edge));
00122 printf("\n");
00123 printf("nodes : %d+%d\n",header->nodesused, header->nodessaved);
00124 printf("edges : %d+%d\n",header->edgesused, header->edgessaved);
00125 printf("============================\n");
00126 }
00127
00128
00129 void
00130 write_node(Dawg_edge *edges, int size, int num, FILE* outfile)
00131 {
00132 #ifdef DEBUG_OUTPUT
00133 int i;
00134 printf("writing %d edges\n",num);
00135 for(i=0; i<num; i++)
00136 {
00137 #ifdef DEBUG_OUTPUT_L2
00138 printf("ptr=%2d t=%d l=%d f=%d chr=%2d (%c)\n",
00139 edges[i].ptr, edges[i].term, edges[i].last,
00140 edges[i].fill, edges[i].chr, edges[i].chr -1 +'a');
00141 #endif
00142 fwrite (edges+i, sizeof(Dawg_edge), 1, outfile);
00143 }
00144 #else
00145 fwrite (edges, size, num, outfile);
00146 #endif
00147 }
00148
00149 #define MAX_STRING_LENGTH 200
00150
00151
00152 #define MAX_EDGES 2000
00153
00154
00155
00156
00157 FILE* global_outfile;
00158 Dict_header global_header;
00159 Hash_table global_hashtable;
00160
00161 char global_stringbuf[MAX_STRING_LENGTH];
00162 char* global_endstring;
00163 char* global_input;
00164 char* global_endofinput;
00165
00166
00167
00168
00169
00170
00171
00172
00173 #ifdef CHECK_RECURSION
00174 int current_rec =0;
00175 int max_rec = 0;
00176 #endif
00177
00178 unsigned int
00179 makenode(char *prefix)
00180 {
00181 int numedges;
00182 Dawg_edge edges[MAX_EDGES];
00183 Dawg_edge *edgeptr = edges;
00184 unsigned *saved_position;
00185
00186 #ifdef CHECK_RECURSION
00187 current_rec++;
00188 if (current_rec > max_rec)
00189 max_rec = current_rec;
00190 #endif
00191
00192 while (prefix == global_endstring)
00193 {
00194
00195 edgeptr->ptr = 0;
00196 edgeptr->term = 0;
00197 edgeptr->last = 0;
00198 edgeptr->fill = 0;
00199 edgeptr->chr = 0;
00200
00201 (*(edgeptr++)).chr = (*global_endstring++ = *global_input++) & DIC_CHAR_MASK;
00202 if (*global_input == '\n')
00203 {
00204 global_header.nwords++;
00205 edgeptr[-1].term = 1;
00206 *global_endstring++ = *global_input++;
00207 if (global_input == global_endofinput)
00208 break;
00209
00210 global_endstring = global_stringbuf;
00211 while(*global_endstring == *global_input)
00212 {
00213 global_endstring++;
00214 global_input++;
00215 }
00216 }
00217
00218 edgeptr[-1].ptr = makenode(prefix + 1);
00219 }
00220
00221 numedges = edgeptr - edges;
00222 if (numedges == 0)
00223 {
00224 #ifdef CHECK_RECURSION
00225 current_rec --;
00226 #endif
00227 return 0;
00228 }
00229
00230 edgeptr[-1].last = 1;
00231
00232 saved_position = (unsigned int*) hash_find (global_hashtable,
00233 (void*)edges,
00234 numedges*sizeof(Dawg_edge));
00235 if (saved_position)
00236 {
00237 global_header.edgessaved += numedges;
00238 global_header.nodessaved++;
00239
00240 #ifdef CHECK_RECURSION
00241 current_rec --;
00242 #endif
00243 return *saved_position;
00244 }
00245 else
00246 {
00247 unsigned int node_pos;
00248
00249 node_pos = global_header.edgesused;
00250 hash_add(global_hashtable,
00251 (void*)edges,numedges*sizeof(Dawg_edge),
00252 (void*)(&global_header.edgesused),sizeof(global_header.edgesused));
00253 global_header.edgesused += numedges;
00254 global_header.nodesused++;
00255 write_node (edges, sizeof(Dawg_edge), numedges, global_outfile);
00256
00257 #ifdef CHECK_RECURSION
00258 current_rec --;
00259 #endif
00260 return node_pos;
00261 }
00262 }
00263
00264
00265
00266
00267 int
00268 main(int argc, char* argv[])
00269 {
00270 unsigned int dicsize;
00271 char *uncompressed;
00272 Dawg_edge rootnode = {0,0,0,0,0};
00273 Dawg_edge specialnode = {0,0,0,0,0};
00274
00275 char* outfilename;
00276 char outfilenamedefault[] = "dict.daw";
00277 clock_t starttime, endtime;
00278
00279 if (argc < 2)
00280 {
00281 fprintf(stderr,"usage: %s uncompressed_dic [compressed_dic]\n",argv[0]);
00282 exit(1);
00283 }
00284
00285 dicsize = file_length (argv[1]);
00286 if (dicsize < 0)
00287 {
00288 fprintf(stderr,"Cannot stat uncompressed dictionary %s\n",argv[1]);
00289 exit(1);
00290 }
00291
00292 outfilename = (argc == 3) ? argv[2] : outfilenamedefault;
00293
00294 if ((global_outfile = fopen (outfilename,"wb")) == NULL)
00295 {
00296 fprintf(stderr,"Cannot open output file %s\n",outfilename);
00297 exit(1);
00298 }
00299
00300 if ((uncompressed = load_uncompressed(argv[1], &dicsize)) == NULL)
00301 {
00302 fprintf(stderr,"Cannot load uncompressed dictionary into memory\n");
00303 exit(1);
00304 }
00305
00306 global_input = uncompressed;
00307 global_endofinput = global_input + dicsize;
00308
00309 #define SCALE 0.6
00310 global_hashtable = hash_init((unsigned int)(dicsize * SCALE));
00311 #undef SCALE
00312
00313 skip_init_header(global_outfile,&global_header);
00314
00315 specialnode.last = 1;
00316 write_node(&specialnode,sizeof(specialnode),1,global_outfile);
00317
00318
00319
00320
00321 starttime=clock();
00322 rootnode.ptr = makenode(global_endstring = global_stringbuf);
00323 endtime=clock();
00324 write_node(&rootnode,sizeof(rootnode),1,global_outfile);
00325
00326 fix_header(global_outfile,&global_header);
00327
00328 print_header_info(&global_header);
00329 hash_destroy(global_hashtable);
00330 free(uncompressed);
00331 fclose(global_outfile);
00332
00333 printf(" Elapsed time is : %f s\n", 1.0*(endtime-starttime) / CLOCKS_PER_SEC);
00334 #ifdef CHECK_RECURSION
00335 printf(" Maximum recursion level reached : %d\n",max_rec);
00336 #endif
00337 return 0;
00338 }
00339
00340