/***************************************************************************** * * $Id$ * Purpose: File Database Maintenance - Build index for request processor * ***************************************************************************** * Copyright (C) 1997-2001 * * Michiel Broek FIDO: 2:280/2802 * Beekmansbos 10 * 1971 BV IJmuiden * the Netherlands * * This file is part of MBSE BBS. * * This BBS is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2, or (at your option) any * later version. * * MBSE BBS is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with MBSE BBS; see the file COPYING. If not, write to the Free * Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. *****************************************************************************/ #include "../lib/libs.h" #include "../lib/structs.h" #include "../lib/records.h" #include "../lib/common.h" #include "../lib/clcomm.h" #include "../lib/dbcfg.h" #include "mbfutil.h" #include "mbfindex.h" extern int do_quiet; /* Supress screen output */ int lastfile; /* Last file number */ typedef struct _Index { struct _Index *next; struct FILEIndex idx; } Findex; static char *wdays[]= {(char *)"Sun",(char *)"Mon",(char *)"Tue", (char *)"Wed",(char *)"Thu",(char *)"Fri", (char *)"Sat"}; static char *months[]= {(char *)"Jan",(char *)"Feb",(char *)"Mar", (char *)"Apr",(char *)"May",(char *)"Jun", (char *)"Jul",(char *)"Aug",(char *)"Sep", (char *)"Oct",(char *)"Nov",(char *)"Dec"}; void tidy_index(Findex **); void tidy_index(Findex **fap) { Findex *tmp, *old; for (tmp = *fap; tmp; tmp = old) { old = tmp->next; free(tmp); } *fap = NULL; } void fill_index(struct FILEIndex, Findex **); void fill_index(struct FILEIndex idx, Findex **fap) { Findex *tmp; tmp = (Findex *)malloc(sizeof(Findex)); tmp->next = *fap; tmp->idx = idx; *fap = tmp; } int comp_index(Findex **, Findex **); void sort_index(Findex **); void sort_index(Findex **fap) { Findex *ta, **vector; size_t n = 0, i; if (*fap == NULL) return; for (ta = *fap; ta; ta = ta->next) n++; vector = (Findex **)malloc(n * sizeof(Findex *)); i = 0; for (ta = *fap; ta; ta = ta->next) vector[i++] = ta; qsort(vector, n, sizeof(Findex *), (int(*)(const void*, const void *))comp_index); (*fap) = vector[0]; i = 1; for (ta = *fap; ta; ta = ta->next) { if (i < n) ta->next = vector[i++]; else ta->next = NULL; } free(vector); return; } int comp_index(Findex **fap1, Findex **fap2) { return strcasecmp((*fap1)->idx.LName, (*fap2)->idx.LName); } /* * Translate ISO 8859-1 characters to named character entities */ void html_massage(char *, char *); void html_massage(char *inbuf, char *outbuf) { char *inptr = inbuf; char *outptr = outbuf; memset(outbuf, 0, sizeof(outbuf)); while (*inptr) { switch ((unsigned char)*inptr) { case '"': sprintf(outptr, """); break; case '&': sprintf(outptr, "&"); break; case '<': sprintf(outptr, "<"); break; case '>': sprintf(outptr, ">"); break; case 160: sprintf(outptr, " "); break; case 161: sprintf(outptr, "¡"); break; case 162: sprintf(outptr, "¢"); break; case 163: sprintf(outptr, "£"); break; case 164: sprintf(outptr, "¤"); break; case 165: sprintf(outptr, "¥"); break; case 166: sprintf(outptr, "¦"); break; case 167: sprintf(outptr, "§"); break; case 168: sprintf(outptr, "¨"); break; case 169: sprintf(outptr, "©"); break; case 170: sprintf(outptr, "ª"); break; case 171: sprintf(outptr, "«"); break; case 172: sprintf(outptr, "¬"); break; case 173: sprintf(outptr, "­"); break; case 174: sprintf(outptr, "®"); break; case 175: sprintf(outptr, "¯"); break; case 176: sprintf(outptr, "°"); break; case 177: sprintf(outptr, "&plumn;"); break; case 178: sprintf(outptr, "²"); break; case 179: sprintf(outptr, "³"); break; case 180: sprintf(outptr, "´"); break; case 181: sprintf(outptr, "µ"); break; case 182: sprintf(outptr, "¶"); break; case 183: sprintf(outptr, "·"); break; case 184: sprintf(outptr, "¸"); break; case 185: sprintf(outptr, "&supl;"); break; case 186: sprintf(outptr, "º"); break; case 187: sprintf(outptr, "»"); break; case 188: sprintf(outptr, "¼"); break; case 189: sprintf(outptr, "½"); break; case 190: sprintf(outptr, "¾"); break; case 191: sprintf(outptr, "¿"); break; case 192: sprintf(outptr, "À"); break; case 193: sprintf(outptr, "Á"); break; case 194: sprintf(outptr, "Â"); break; case 195: sprintf(outptr, "Ã"); break; case 196: sprintf(outptr, "Ä"); break; case 197: sprintf(outptr, "Å"); break; case 198: sprintf(outptr, "Æ"); break; case 199: sprintf(outptr, "Ç"); break; case 200: sprintf(outptr, "È"); break; case 201: sprintf(outptr, "É"); break; case 202: sprintf(outptr, "Ê"); break; case 203: sprintf(outptr, "Ë"); break; case 204: sprintf(outptr, "Ì"); break; case 205: sprintf(outptr, "Í"); break; case 206: sprintf(outptr, "Î"); break; case 207: sprintf(outptr, "Ï"); break; case 208: sprintf(outptr, "Ð"); break; case 209: sprintf(outptr, "Ñ"); break; case 210: sprintf(outptr, "Ò"); break; case 211: sprintf(outptr, "Ó"); break; case 212: sprintf(outptr, "Ô"); break; case 213: sprintf(outptr, "Õ"); break; case 214: sprintf(outptr, "Ö"); break; case 215: sprintf(outptr, "×"); break; case 216: sprintf(outptr, "Ø"); break; case 217: sprintf(outptr, "Ù"); break; case 218: sprintf(outptr, "Ú"); break; case 219: sprintf(outptr, "Û"); break; case 220: sprintf(outptr, "Ü"); break; case 221: sprintf(outptr, "Ý"); break; case 222: sprintf(outptr, "Þ"); break; case 223: sprintf(outptr, "ß"); break; case 224: sprintf(outptr, "à"); break; case 225: sprintf(outptr, "á"); break; case 226: sprintf(outptr, "â"); break; case 227: sprintf(outptr, "ã"); break; case 228: sprintf(outptr, "ä"); break; case 229: sprintf(outptr, "å"); break; case 230: sprintf(outptr, "æ"); break; case 231: sprintf(outptr, "ç"); break; case 232: sprintf(outptr, "è"); break; case 233: sprintf(outptr, "é"); break; case 234: sprintf(outptr, "ê"); break; case 235: sprintf(outptr, "ë"); break; case 236: sprintf(outptr, "ì"); break; case 237: sprintf(outptr, "í"); break; case 238: sprintf(outptr, "î"); break; case 239: sprintf(outptr, "ï"); break; case 240: sprintf(outptr, "ð"); break; case 241: sprintf(outptr, "ñ"); break; case 242: sprintf(outptr, "ò"); break; case 243: sprintf(outptr, "ó"); break; case 244: sprintf(outptr, "ô"); break; case 245: sprintf(outptr, "õ"); break; case 246: sprintf(outptr, "ö"); break; case 247: sprintf(outptr, "÷"); break; case 248: sprintf(outptr, "ø"); break; case 249: sprintf(outptr, "ù"); break; case 250: sprintf(outptr, "ú"); break; case 251: sprintf(outptr, "û"); break; case 252: sprintf(outptr, "ü"); break; case 253: sprintf(outptr, "ý"); break; case 254: sprintf(outptr, "þ"); break; case 255: sprintf(outptr, "ÿ"); break; default: *outptr++ = *inptr; *outptr = '\0'; break; } while (*outptr) outptr++; inptr++; } *outptr = '\0'; } char *rfcdate(time_t); char *rfcdate(time_t now) { static char buf[40]; struct tm ptm; ptm = *gmtime(&now); sprintf(buf,"%s, %02d %s %04d %02d:%02d:%02d GMT", wdays[ptm.tm_wday], ptm.tm_mday, months[ptm.tm_mon], ptm.tm_year + 1900, ptm.tm_hour, ptm.tm_min, ptm.tm_sec); return(buf); } void pagelink(FILE *, char *, int, int); void pagelink(FILE *fa, char *Path, int inArea, int Current) { char nr[20]; fprintf(fa, "
\n"); if ((Current >= CFG.www_files_page) && (inArea >= CFG.www_files_page)) { if (((Current / CFG.www_files_page) - 1) > 0) sprintf(nr, "%d", (Current / CFG.www_files_page) -1); else nr[0] = '\0'; fprintf(fa, "\"%s\"%s \n", CFG.www_url, CFG.www_link2ftp, Path+strlen(CFG.ftp_base), nr, CFG.www_icon_prev, CFG.www_name_prev, CFG.www_name_prev); } fprintf(fa, "\"%s\"%s \n", CFG.www_url, CFG.www_icon_home, CFG.www_name_home, CFG.www_name_home); fprintf(fa, "\"%s\"%s\n", CFG.www_url, CFG.www_link2ftp, CFG.www_icon_back, CFG.www_name_back, CFG.www_name_back); if ((Current < (inArea - CFG.www_files_page)) && (inArea >= CFG.www_files_page)) { fprintf(fa, " \"%s\"%s\n", CFG.www_url, CFG.www_link2ftp, Path+strlen(CFG.ftp_base), (Current / CFG.www_files_page) + 1, CFG.www_icon_next, CFG.www_name_next, CFG.www_name_next); } fprintf(fa, "

\n"); } FILE *newpage(char *, char *, time_t, int, int); FILE *newpage(char *Path, char *Name, time_t later, int inArea, int Current) { char linebuf[1024], outbuf[1024]; static FILE* fa; lastfile = Current; if (Current) sprintf(linebuf, "%s/index%d.temp", Path, Current / CFG.www_files_page); else sprintf(linebuf, "%s/index.temp", Path); if ((fa = fopen(linebuf, "w")) == NULL) { WriteError("$Can't create %s", linebuf); } else { sprintf(linebuf, "%s", Name); html_massage(linebuf, outbuf); fprintf(fa, "\n"); fprintf(fa, "\n", rfcdate(later)); fprintf(fa, "\n"); fprintf(fa, "\n", CFG.www_charset); fprintf(fa, "\n", CFG.www_author, outbuf); fprintf(fa, "%s\n", outbuf); fprintf(fa, "\n", CFG.www_url); fprintf(fa, "\n\n\n"); pagelink(fa, Path, inArea, Current); fprintf(fa, "

File index of %s

\n", outbuf); fprintf(fa, "\n"); fprintf(fa, "\n"); return fa; } return NULL; } void closepage(FILE *, char *, int, int); void closepage(FILE *fa, char *Path, int inArea, int Current) { char *temp1, *temp2; if (fa == NULL) return; temp1 = calloc(PATH_MAX, sizeof(char)); temp2 = calloc(PATH_MAX, sizeof(char)); fprintf(fa, "
Nr.FilenameDateSizeDownloadsDescription

\n"); pagelink(fa, Path, inArea, lastfile); fprintf(fa, "\n"); fclose(fa); if (lastfile) { sprintf(temp1, "%s/index%d.html", Path, lastfile / CFG.www_files_page); sprintf(temp2, "%s/index%d.temp", Path, lastfile / CFG.www_files_page); } else { sprintf(temp1, "%s/index.html", Path); sprintf(temp2, "%s/index.temp", Path); } rename(temp2, temp1); free(temp1); free(temp2); chmod(temp1, 0644); fa = NULL; } /* * Build a sorted index for the file request processor. * Build html index pages for download. */ void Index(void) { FILE *pAreas, *pFile, *pIndex, *fa, *fm, *fp; long i, iAreas, iAreasNew = 0, record, iSize = 0L, aSize = 0; int iTotal = 0, AreaNr = 0, j, z, x = 0, Areas = 0; int Total = 0, aTotal = 0, inArea = 0, filenr; int fbAreas = 0, fbFiles = 0; char *sAreas, *fAreas, *newdir = NULL, *sIndex, *fn, *temp; char linebuf[1024], outbuf[1024]; time_t last = 0L, later; Findex *fdx = NULL; Findex *tmp; struct FILEIndex idx; sAreas = calloc(PATH_MAX, sizeof(char)); fAreas = calloc(PATH_MAX, sizeof(char)); sIndex = calloc(PATH_MAX, sizeof(char)); fn = calloc(PATH_MAX, sizeof(char)); temp = calloc(PATH_MAX, sizeof(char)); later = time(NULL) + 86400; IsDoing("Index files"); if (!do_quiet) { colour(3, 0); printf("Create index files...\n"); } sprintf(sAreas, "%s/etc/fareas.data", getenv("MBSE_ROOT")); if ((pAreas = fopen (sAreas, "r")) == NULL) { WriteError("$Can't open %s", sAreas); die(0); } sprintf(sIndex, "%s/etc/request.index", getenv("MBSE_ROOT")); if ((pIndex = fopen(sIndex, "w")) == NULL) { WriteError("$Can't create %s", sIndex); die(0); } fread(&areahdr, sizeof(areahdr), 1, pAreas); fseek(pAreas, 0, SEEK_END); iAreas = (ftell(pAreas) - areahdr.hdrsize) / areahdr.recsize; /* * Check if we are able to create the index.html pages in the * download directories. */ if (strlen(CFG.ftp_base) && strlen(CFG.www_url) && strlen(CFG.www_author) && strlen(CFG.www_charset)) { sprintf(fn, "%s/index.temp", CFG.ftp_base); if ((fm = fopen(fn, "w")) == NULL) { Syslog('+', "Can't open %s, skipping html pages creation", fn); } } else { fm = NULL; Syslog('+', "FTP/HTML not defined, skipping html pages creation"); } if (fm) { /* * Because these web pages are dynamic, ie. they change everytime you * receive new files and recreates these pages, extra HTTP headers are * send to the client about these pages. It forbids proxy servers to * cache these pages. The pages will expire within 24 hours. The pages * also have an author name, this is the bbs name, and a content * description for search engines. Automatic advertising. */ sprintf(linebuf, "File areas at %s", CFG.bbs_name); html_massage(linebuf, outbuf); fprintf(fm, "\n"); fprintf(fm, "\n", rfcdate(later)); fprintf(fm, "\n"); fprintf(fm, "\n", CFG.www_charset); fprintf(fm, "\n", CFG.www_author, outbuf); fprintf(fm, "%s\n", outbuf); fprintf(fm, "\n", CFG.www_url); fprintf(fm, "\n\n\n"); fprintf(fm, "

%s

\n", outbuf); fprintf(fm, "\n"); fprintf(fm, "\n"); } for (i = 1; i <= iAreas; i++) { fseek(pAreas, ((i-1) * areahdr.recsize) + areahdr.hdrsize, SEEK_SET); fread(&area, areahdr.recsize, 1, pAreas); if (area.Available) { if (!diskfree(CFG.freespace)) die(101); if (!do_quiet) { printf("\r%4ld => %-44s \b\b\b\b", i, area.Name); fflush(stdout); } /* * Check if download directory exists, * if not, create the directory. */ if (access(area.Path, W_OK) == -1) { Syslog('!', "Create dir: %s", area.Path); newdir = xstrcpy(area.Path); newdir = xstrcat(newdir, (char *)"/"); mkdirs(newdir); free(newdir); newdir = NULL; } sprintf(fAreas, "%s/fdb/fdb%ld.data", getenv("MBSE_ROOT"), i); /* * Open the file database, if it doesn't exist, * create an empty one. */ if ((pFile = fopen(fAreas, "r+")) == NULL) { Syslog('!', "Creating new %s", fAreas); if ((pFile = fopen(fAreas, "a+")) == NULL) { WriteError("$Can't create %s", fAreas); die(0); } } /* * Create file request index if requests are allowed in this area. */ if (area.FileReq) { /* * Now start creating the unsorted index. */ record = 0; while (fread(&file, sizeof(file), 1, pFile) == 1) { iTotal++; if ((iTotal % 10) == 0) Marker(); memset(&idx, 0, sizeof(idx)); sprintf(idx.Name, "%s", tu(file.Name)); sprintf(idx.LName, "%s", tu(file.LName)); idx.AreaNum = i; idx.Record = record; fill_index(idx, &fdx); record++; } iAreasNew++; } /* if area.Filereq */ /* * Create files.bbs */ if (strlen(area.FilesBbs)) strcpy(temp, area.FilesBbs); else sprintf(temp, "%s/files.bbs", area.Path); if ((fp = fopen(temp, "w")) == NULL) { WriteError("$Can't create %s", temp); } else { fseek(pFile, 0, SEEK_SET); fbAreas++; while (fread(&file, sizeof(file), 1, pFile) == 1) { if ((!file.Deleted) && (!file.Missing)) { fbFiles++; fprintf(fp, "%-12s [%ld] %s\r\n", file.Name, file.TimesDL + file.TimesFTP + file.TimesReq, file.Desc[0]); for (j = 1; j < 25; j++) if (strlen(file.Desc[j])) fprintf(fp, " +%s\r\n", file.Desc[j]); } } fclose(fp); } /* * Create 00index file and index.html pages in each available download area. */ if (!area.CDrom && fm && (strncmp(CFG.ftp_base, area.Path, strlen(CFG.ftp_base)) == 0)) { sprintf(temp, "%s/00index", area.Path); if ((fp = fopen(temp, "w")) == NULL) { WriteError("$Can't create %s", temp); } else { fseek(pFile, 0, SEEK_SET); Areas++; inArea = 0; while (fread(&file, sizeof(file), 1, pFile) == 1) { if ((!file.Deleted) && (!file.Missing)) inArea++; } fseek(pFile, 0, SEEK_SET); aSize = 0L; aTotal = 0; last = 0L; fa = newpage(area.Path, area.Name, later, inArea, aTotal); while (fread(&file, sizeof(file), 1, pFile) == 1) { if ((!file.Deleted) && (!file.Missing)) { /* * The next is to reduce system load */ x++; Total++; aTotal++; if (CFG.slow_util && do_quiet && ((x % 3) == 0)) usleep(1); for (z = 0; z <= 25; z++) { if (strlen(file.Desc[z])) { if (z == 0) fprintf(fp, "%-12s %7luK %s ", file.Name, (long)(file.Size / 1024), StrDateDMY(file.UploadDate)); else fprintf(fp, " "); if ((file.Desc[z][0] == '@') && (file.Desc[z][1] == 'X')) fprintf(fp, "%s\n", file.Desc[z]+4); else fprintf(fp, "%s\n", file.Desc[z]); } } fprintf(fa, "", aTotal); /* * Check if this is a .gif or .jpg file, if so then * check if a thumbnail file exists. If not try to * create a thumbnail file to add to the html listing. */ if (strstr(file.LName, ".gif") || strstr(file.LName, ".jpg")) { sprintf(linebuf, "%s/%s", area.Path, file.LName); sprintf(outbuf, "%s/.%s", area.Path, file.LName); if (file_exist(outbuf, R_OK)) { if ((j = execute(CFG.www_convert, linebuf, outbuf, (char *)"/dev/null", (char *)"/dev/null", (char *)"/dev/null"))) { Syslog('+', "Failed to create thumbnail for %s, rc=% d", file.LName, j); } } fprintf(fa, ""); } else { fprintf(fa, "", CFG.www_url, CFG.www_link2ftp, area.Path+strlen(CFG.ftp_base), file.LName, file.LName); } fprintf(fa, "", StrDateDMY(file.FileDate)); fprintf(fa, "", (long)(file.Size / 1024)); fprintf(fa, "", file.TimesDL + file.TimesFTP + file.TimesReq); fprintf(fa, "\n"); aSize += file.Size; iSize += file.Size; if (file.FileDate > last) last = file.FileDate; if ((aTotal % CFG.www_files_page) == 0) { closepage(fa, area.Path, inArea, aTotal); fa = newpage(area.Path, area.Name, later, inArea, aTotal); } } /* if (!file.deleted) */ } closepage(fa, area.Path, inArea, aTotal); fclose(fp); /* * If the time before there were more files in this area then now, * the number of html pages may de decreased. We try to delete these * files if they should exist. */ filenr = lastfile / CFG.www_files_page; while (TRUE) { filenr++; sprintf(linebuf, "%s/index%d.html", area.Path, filenr); if (unlink(linebuf)) break; Syslog('+', "Removed obsolete %s", linebuf); } fprintf(fm, "", AreaNr, CFG.www_url, CFG.www_link2ftp, area.Path+strlen(CFG.ftp_base), area.Name); fprintf(fm, "", aTotal); if (aSize > 1048576) fprintf(fm, "", aSize / 1048576); else fprintf(fm, "", aSize / 1024); if (last == 0L) fprintf(fm, "\n"); else fprintf(fm, "\n", StrDateDMY(last)); } } fclose(pFile); } /* if area.Available */ } if (fm) { fprintf(fm, "\n", Total, iSize / 1048576); fprintf(fm, "
AreaDescriptionFilesTotal sizeLast added
%d", CFG.www_url, CFG.www_link2ftp, area.Path+strlen(CFG.ftp_base), file.LName); fprintf(fa, "\"%s\"", CFG.www_url, CFG.www_link2ftp, area.Path+strlen(CFG.ftp_base), file.LName, file.LName); fprintf(fa, "%s%s%lu Kb.%8ld
");
			    for (j = 0; j < 25; j++)
				if (strlen(file.Desc[j])) {
				    if (j)
					fprintf(fa, "\n");
				    sprintf(linebuf, "%s", strkconv(file.Desc[j], CHRS_DEFAULT_FTN, CHRS_DEFAULT_RFC));
				    html_massage(linebuf, outbuf);
				    fprintf(fa, "%s", outbuf);
				}
			    fprintf(fa, "
%d%s%d%ld Mb.%ld Kb. 
%s
 Total%d%ld Mb. 

\n"); fprintf(fm, "\"%s\"%s\n", CFG.www_icon_home, CFG.www_name_home, CFG.www_name_home); fprintf(fm, "\n"); fclose(fm); sprintf(linebuf, "%s/index.html", CFG.ftp_base); rename(fn, linebuf); chmod(linebuf, 0644); } fclose(pAreas); sort_index(&fdx); for (tmp = fdx; tmp; tmp = tmp->next) fwrite(&tmp->idx, sizeof(struct FILEIndex), 1, pIndex); fclose(pIndex); tidy_index(&fdx); Syslog('+', "Index Areas [%5d] Files [%5d]", iAreasNew, iTotal); Syslog('+', "Files Areas [%5d] Files [%5d]", fbAreas, fbFiles); Syslog('+', "HTML Areas [%5d] Files [%5d]", Areas, Total); if (!do_quiet) { printf("\r \r"); fflush(stdout); } free(sIndex); free(sAreas); free(fAreas); free(fn); free(temp); RemoveSema((char *)"reqindex"); }