deb-mbse/mbfido/scannews.c

/*****************************************************************************
 *
 * $Id$
 * Purpose ...............: Scan for new News
 *
 *****************************************************************************
 * Copyright (C) 1997-2002
 *   
 * Michiel Broek		FIDO:		2:280/2802
 * Beekmansbos 10
 * 1971 BV IJmuiden
 * the Netherlands
 *
 * This file is part of MBSE BBS.
 *
 * This BBS is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2, or (at your option) any
 * later version.
 *
 * MBSE BBS is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with MBSE BBS; see the file COPYING.  If not, write to the Free
 * Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *****************************************************************************/

#include "../config.h"
#include "../lib/libs.h"
#include "../lib/memwatch.h"
#include "../lib/structs.h"
#include "../lib/users.h"
#include "../lib/records.h"
#include "../lib/common.h"
#include "../lib/clcomm.h"
#include "../lib/mbinet.h"
#include "../lib/dbdupe.h"
#include "../lib/dbnode.h"
#include "../lib/dbmsgs.h"
#include "../lib/msg.h"
#include "../lib/msgtext.h"
#include "mkftnhdr.h"
#include "hash.h"
#include "rollover.h"
#include "storeecho.h"
#include "rfc2ftn.h"
#include "scannews.h"


#define MAXHDRSIZE 2048
#define	MAXSEEN 70
#define	MAXPATH 73


/*
 *  Global variables
 */
POverview	xoverview = NULL;
int		marker = 0;



/*
 *  External variables
 */
extern	int	do_quiet;
extern	int	do_learn;
extern	int	news_in;
extern	int	news_imp;
extern	int	news_dupe;
extern	int	echo_out;
extern	int	echo_in;
extern	int	do_flush;
extern	char	*replyaddr;



/*
 *  Internal functions
 */
int		do_one_group(List **, char *, char *, int);
int		get_xover(char *, long, long, List **);
int		get_xoverview(void);
void		tidy_artlist(List **);
void		fill_artlist(List **, char *, long, int);
void		Marker(void);
int		get_article(char *, char *);



void tidy_artlist(List **fdp)
{
	List	*tmp, *old;

	for (tmp = *fdp; tmp; tmp = old) {
		old = tmp->next;
		free(tmp);
	}
	*fdp = NULL;
}



/*
 * Add article to the list
 */
void fill_artlist(List **fdp, char *id, long nr, int dupe)
{
	List	**tmp;

	Syslog('N', "Fill %s %ld %s", id, nr, dupe ? "Dupe":"New msg");

	for (tmp = fdp; *tmp; tmp = &((*tmp)->next));
	*tmp = (List *)malloc(sizeof(List));
	(*tmp)->next = NULL;
	sprintf((*tmp)->msgid, "%s", id);
	(*tmp)->nr = nr;
	(*tmp)->isdupe = dupe;
}



void Marker(void)
{
        if (do_quiet)
                return;

        switch (marker) {
                case 0: printf(">---");
                        break;

                case 1: printf(">>--");
                        break;

                case 2: printf(">>>-");
                        break;

                case 3: printf(">>>>");
                        break;

                case 4: printf("<>>>");
                        break;

                case 5: printf("<<>>");
                        break;

                case 6: printf("<<<>");
                        break;

                case 7: printf("<<<<");
                        break;

                case 8: printf("-<<<");
                        break;

                case 9: printf("--<<");
                        break;

                case 10:printf("---<");
                        break;

                case 11:printf("----");
                        break;
        }
        printf("\b\b\b\b");
        fflush(stdout);

        if (marker < 11)
                marker++;
        else
                marker = 0;
}



/*
 *  Scan for new news available at the nntp server.
 */
void ScanNews(void)
{
	List			*art = NULL;
	POverview		tmp, old;
	FILE			*pAreas;
	char			*sAreas;
	struct msgareashdr	Msgshdr;
	struct msgareas		Msgs;

	IsDoing((char *)"Scan News");
	if (nntp_connect() == -1) {
		WriteError("Can't connect to newsserver");
		return;
	}
	if (get_xoverview()) {
		return;
	}

	if (!do_quiet) {
		colour(10, 0);
		printf("Scan for new news articles\n");
	}

	sAreas = calloc(PATH_MAX, sizeof(char));
	sprintf(sAreas, "%s/etc/mareas.data", getenv("MBSE_ROOT"));
	if(( pAreas = fopen (sAreas, "r")) == NULL) {
		WriteError("$Can't open Messages Areas File.");
		return;
	}
	fread(&Msgshdr, sizeof(Msgshdr), 1, pAreas);

	while (fread(&Msgs, Msgshdr.recsize, 1, pAreas) == 1) {
		fseek(pAreas, Msgshdr.syssize, SEEK_CUR);
		if ((Msgs.Active) && strlen(Msgs.Newsgroup)) {
			if (IsSema((char *)"upsalarm")) {
				Syslog('+', "Detected upsalarm semafore, aborting newsscan");
				break;
			}
			Syslog('n', "Scan newsgroup: %s", Msgs.Newsgroup);
			if (!do_quiet) {
				colour(3, 0);
				printf("\r%-40s", Msgs.Newsgroup);
				fflush(stdout);
			}
			Nopper();
			if (do_one_group(&art, Msgs.Newsgroup, Msgs.Tag, Msgs.MaxArticles) == RETVAL_ERROR)
				break;
			/*
			 * To be safe, update the dupes database after each area.
			 */
			CloseDupes();
		}
	}
	fclose(pAreas);
	free(sAreas);

	for (tmp = xoverview; tmp; tmp = old) {
		old = tmp->next;
		if (tmp->header)
			free(tmp->header);
		if (tmp->field)
			free(tmp->field);
		free(tmp);
	}
	do_flush = TRUE;
	if (!do_quiet)
		printf("\r                                                    \r");
}



int do_one_group(List **art, char *grpname, char *ftntag, int maxarticles)
{
	List	*tmp;
	char	temp[128], *resp;
	int	retval, fetched = 0;
	long	total, start, end;

	Syslog('N', "do_one_group(%s, %s)", grpname, ftntag);
	IsDoing((char *)"Scan %s", grpname);
	sprintf(temp, "GROUP %s\r\n", grpname);
	nntp_send(temp);
	resp = nntp_receive();
	retval = atoi(strtok(resp, " "));
	if (retval != 211) {
		if (retval == 411) {
			WriteError("No such newsgroup: %s", grpname);
			return RETVAL_UNEXPECTEDANS;
		}
		WriteError("Unknown response %d to GROUP command", retval);
		return RETVAL_ERROR;
	}

	total = atol(strtok(NULL, " "));
	start = atol(strtok(NULL, " "));
	end   = atol(strtok(NULL, " '\0'"));
	Syslog('n', "GROUP total %d, start %d, end %d, max %d", total, start, end, maxarticles);
	if ((maxarticles) && (total > maxarticles)) {
	    start = end - maxarticles;
	    total = maxarticles;
	    Syslog('n', "NEW:  total %d, start %d, end %d", total, start, end);
	}
	if (!total) {
		Syslog('N', "No articles");
		return RETVAL_NOARTICLES;
	}

	retval = get_xover(grpname, start, end, art);
	if (retval != RETVAL_OK) {
		tidy_artlist(art);
		return retval;
	}

	if (!do_learn) {
		for (tmp = *art; tmp; tmp = tmp->next) {
			if (!tmp->isdupe) {
				/*
				 *  If the message isn't a dupe, it must be new for us.
				 */
				get_article(tmp->msgid, ftntag);
				fetched++;
			}
		}
	}

	tidy_artlist(art);

	if ((maxarticles) && (fetched == maxarticles))
	    Syslog('!', "Warning: the max. articles value in newsgroup %s might be to low", grpname);

	return RETVAL_OK;
}



int get_article(char *msgid, char *ftntag)
{
	char	cmd[81], *resp;
	int	retval, done = FALSE;
	FILE	*fp = NULL, *dp;
	char	dpath[PATH_MAX];

	Syslog('n', "Get article %s, %s", msgid, ftntag);
	if (!SearchMsgs(ftntag)) {
		WriteError("Search message area %s failed", ftntag);
		return RETVAL_ERROR;
	}

	sprintf(dpath, "%s/tmp/scannews.last", getenv("MBSE_ROOT"));
	dp = fopen(dpath, "w");

	IsDoing("Article %d", (news_in + 1));
	sprintf(cmd, "ARTICLE %s\r\n", msgid);
	fprintf(dp, "ARTICLE %s\n", msgid);
	nntp_send(cmd);
	resp = nntp_receive();
	fprintf(dp, "%s\n", resp);
	retval = atoi(strtok(resp, " "));
	switch (retval) {
		case 412:	WriteError("No newsgroup selected");
				return RETVAL_UNEXPECTEDANS;
		case 420:	WriteError("No current article has been selected");
				return RETVAL_UNEXPECTEDANS;
		case 423:	WriteError("No such article in this group");
				return RETVAL_UNEXPECTEDANS;
		case 430:	WriteError("No such article found");
				return RETVAL_UNEXPECTEDANS;
		case 220:	if ((fp = tmpfile()) == NULL) {
					WriteError("$Can't open tmpfile");
					return RETVAL_UNEXPECTEDANS;
				}
				while (done == FALSE) {
					resp = nntp_receive();
					fwrite(resp, strlen(resp), 1, dp);
					fprintf(dp, "\n");
					fflush(dp);
					if ((strlen(resp) == 1) && (strcmp(resp, ".") == 0)) {
						done = TRUE;
					} else {
						fwrite(resp, strlen(resp), 1, fp);
						fputc('\n', fp);
					}
				}
				break;
	}

	IsDoing("Article %d", (news_in));
	retval = rfc2ftn(fp, NULL);
	fclose(fp);
	fclose(dp);
	return retval;
}



int get_xover(char *grpname, long startnr, long endnr, List **art)
{
	char		cmd[81], *ptr, *ptr2, *resp, *p;
	int		retval, dupe, done = FALSE;
	long		nr;
	unsigned long	crc;
	POverview	pov;

	sprintf(cmd, "XOVER %ld-%ld\r\n", startnr, endnr);
	if ((retval = nntp_cmd(cmd, 224))) {
		switch (retval) {
			case 412:
				WriteError("No newsgroup selected");
				return RETVAL_NOXOVER;
			case 502:
				WriteError("Permission denied");
				return RETVAL_NOXOVER;
			case 420:
				Syslog('n', "No articles in group %s", grpname);
				return RETVAL_OK;
		}
	}

	while (done == FALSE) {
		resp = nntp_receive();
		if ((strlen(resp) == 1) && (strcmp(resp, ".") == 0)) {
			done = TRUE;
		} else {
			Marker();
			Nopper();
			pov = xoverview;
			ptr = resp;
			ptr2 = ptr;

			/*
			 * First item is the message number.
			 */
			while (*ptr2 != '\0' && *ptr2 != '\t')
				ptr2++;
			if (*ptr2 != '\0')
				*(ptr2) = '\0';
			nr = atol(ptr);
			ptr = ptr2;
			ptr++;

			/*
			 * Search the message-id
			 */
			while (*ptr != '\0' && pov != NULL && strcmp(pov->header, "Message-ID:") != 0) {
				/*
				 * goto the next field, past the tab.
				 */
				pov = pov->next;

				while (*ptr != '\t' && *ptr != '\0')
					ptr++;
				if (*ptr != '\0')
					ptr++;
			}
			if (*ptr != '\0' && pov != NULL) {
				/*
				 * Found it, now find start of msgid
				 */
				while (*ptr != '\0' && *ptr != '<')
					ptr++;
				if(ptr != '\0') {
					ptr2 = ptr;
					while(*ptr2 != '\0' && *ptr2 != '>')
						ptr2++;
					if (*ptr2 != '\0') {
						*(ptr2+1) = '\0';
						p = xstrcpy(ptr);
						p = xstrcat(p, grpname);
						crc = str_crc32(p);
						dupe = CheckDupe(crc, D_NEWS, CFG.nntpdupes);
						fill_artlist(art, ptr, nr, dupe);
						free(p);
						if (CFG.slow_util && do_quiet)
							usleep(1);
					}
				}
			}
		}
	}

	return RETVAL_OK;
}



int get_xoverview(void)
{
	int		retval, len, full, done = FALSE;
	char		*resp;
	POverview	tmp, curptr = NULL;

	Syslog('n', "Getting overview format list");
	if ((retval = nntp_cmd((char *)"LIST overview.fmt\r\n", 215)) == 0) {
		while (done == FALSE) {
			resp = nntp_receive();
			if ((strcmp(resp, ".") == 0) && (strlen(resp) == 1)) {
				done = TRUE;
			} else {
				len = strlen(resp);
				/*
				 * Check for the full flag, which means the field name
				 * is in the xover string.
				 */
				full = (strstr(resp, ":full") == NULL) ? FALSE : TRUE;
				/*
				 * Now get rid of everything back to :
				 */
				while (resp[len] != ':')
					resp[len--] = '\0';
				len++;

				tmp = malloc(sizeof(Overview));
				tmp->header = calloc(len + 1, sizeof(char));
				strncpy(tmp->header, resp, len);
				tmp->header[len] = '\0';
				tmp->next = NULL;
				tmp->field = NULL;
				tmp->fieldlen = 0;
				tmp->full = full;

				if (curptr == NULL) {
					/* at head of list */
					curptr = tmp;
					xoverview = tmp;
				} else {
					/* add to linked list */
					curptr->next = tmp;
					curptr = tmp;
				}
			}
		}

		if ((tmp = xoverview) != NULL) {
			Syslog('N', "--Xoverview.fmt list");
			while (tmp != NULL) {
				if (tmp->header != NULL) {
					Syslog('N', "item = %s -- full = %s", tmp->header, tmp->full ? "True":"False");
				}
				tmp = tmp->next;
			}
		}
	} else {
		return 1;
	}
	return 0;
}