deb-mbse/mbfido/scannews.c

/*****************************************************************************
 *
 * $Id$
 * Purpose ...............: Scan for new News
 *
 *****************************************************************************
 * Copyright (C) 1997-2004
 *   
 * Michiel Broek		FIDO:		2:280/2802
 * Beekmansbos 10
 * 1971 BV IJmuiden
 * the Netherlands
 *
 * This file is part of MBSE BBS.
 *
 * This BBS is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2, or (at your option) any
 * later version.
 *
 * MBSE BBS is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with MBSE BBS; see the file COPYING.  If not, write to the Free
 * Software Foundation, 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
 *****************************************************************************/

#include "../config.h"
#include "../lib/mbselib.h"
#include "../lib/users.h"
#include "../lib/mbinet.h"
#include "../lib/mbsedb.h"
#include "../lib/msg.h"
#include "../lib/msgtext.h"
#include "mkftnhdr.h"
#include "hash.h"
#include "rollover.h"
#include "storeecho.h"
#include "rfc2ftn.h"
#include "scannews.h"


#define MAXHDRSIZE 2048
#define	MAXSEEN 70
#define	MAXPATH 73


/*
 *  Global variables
 */
POverview	xoverview = NULL;
int		marker = 0;



/*
 *  External variables
 */
extern	int	do_quiet;
extern	int	do_learn;
extern	int	news_in;
extern	int	news_imp;
extern	int	news_dupe;
extern	int	echo_out;
extern	int	echo_in;
extern	int	do_flush;
extern	char	*replyaddr;



/*
 *  Internal functions
 */
int		do_one_group(List **, char *, char *, int);
int		get_xover(char *, long, long, List **);
int		get_xoverview(void);
void		tidy_artlist(List **);
void		fill_artlist(List **, char *, long, int);
void		Marker(void);
int		get_article(char *, char *);



void tidy_artlist(List **fdp)
{
	List	*tmp, *old;

	for (tmp = *fdp; tmp; tmp = old) {
		old = tmp->next;
		free(tmp);
	}
	*fdp = NULL;
}



/*
 * Add article to the list
 */
void fill_artlist(List **fdp, char *id, long nr, int dupe)
{
	List	**tmp;

	Syslog('M', "Fill %s %ld %s", id, nr, dupe ? "Dupe":"New msg");

	for (tmp = fdp; *tmp; tmp = &((*tmp)->next));
	*tmp = (List *)malloc(sizeof(List));
	(*tmp)->next = NULL;
	sprintf((*tmp)->msgid, "%s", id);
	(*tmp)->nr = nr;
	(*tmp)->isdupe = dupe;
}



void Marker(void)
{
        if (do_quiet)
                return;

        switch (marker) {
                case 0: printf(">---");
                        break;

                case 1: printf(">>--");
                        break;

                case 2: printf(">>>-");
                        break;

                case 3: printf(">>>>");
                        break;

                case 4: printf("<>>>");
                        break;

                case 5: printf("<<>>");
                        break;

                case 6: printf("<<<>");
                        break;

                case 7: printf("<<<<");
                        break;

                case 8: printf("-<<<");
                        break;

                case 9: printf("--<<");
                        break;

                case 10:printf("---<");
                        break;

                case 11:printf("----");
                        break;
        }
        printf("\b\b\b\b");
        fflush(stdout);

        if (marker < 11)
                marker++;
        else
                marker = 0;
}



/*
 *  Scan for new news available at the nntp server.
 */
void ScanNews(void)
{
    List		*art = NULL;
    POverview		tmp, old;
    FILE		*pAreas;
    char		*sAreas;
    struct msgareashdr	Msgshdr;
    struct msgareas	Msgs;

    IsDoing((char *)"Scan News");
    if (nntp_connect() == -1) {
	WriteError("Can't connect to newsserver");
	return;
    }
    if (get_xoverview()) {
	return;
    }

    if (!do_quiet) {
	colour(10, 0);
	printf("Scan for new news articles\n");
    }

    sAreas = calloc(PATH_MAX, sizeof(char));
    sprintf(sAreas, "%s/etc/mareas.data", getenv("MBSE_ROOT"));
    if(( pAreas = fopen (sAreas, "r")) == NULL) {
	WriteError("$Can't open Messages Areas File.");
	return;
    }
    fread(&Msgshdr, sizeof(Msgshdr), 1, pAreas);

    while (fread(&Msgs, Msgshdr.recsize, 1, pAreas) == 1) {
	fseek(pAreas, Msgshdr.syssize, SEEK_CUR);
#ifdef USE_NEWSGATE
	if ((Msgs.Active) && strlen(Msgs.Newsgroup)) {
#else
	if ((Msgs.Active) && strlen(Msgs.Newsgroup) && (Msgs.Type == NEWS)) {
#endif
	    if (IsSema((char *)"upsalarm")) {
		Syslog('+', "Detected upsalarm semafore, aborting newsscan");
		break;
	    }
	    Syslog('m', "Scan newsgroup: %s", Msgs.Newsgroup);
	    if (!do_quiet) {
		colour(3, 0);
		printf("\r%-40s", Msgs.Newsgroup);
		fflush(stdout);
	    }
	    Nopper();
	    if (do_one_group(&art, Msgs.Newsgroup, Msgs.Tag, Msgs.MaxArticles) == RETVAL_ERROR)
		break;
	    /*
	     * To be safe, update the dupes database after each area.
	     */
	    CloseDupes();
	}
    }
    fclose(pAreas);
    free(sAreas);

    for (tmp = xoverview; tmp; tmp = old) {
	old = tmp->next;
	if (tmp->header)
	    free(tmp->header);
	if (tmp->field)
	    free(tmp->field);
	free(tmp);
    }

    nntp_close();

    do_flush = TRUE;
    if (!do_quiet)
	printf("\r                                                    \r");
}



int do_one_group(List **art, char *grpname, char *ftntag, int maxarticles)
{
    List    *tmp;
    char    temp[128], *resp;
    int	    retval, fetched = 0;
    long    total, start, end;

    Syslog('m', "do_one_group(%s, %s)", grpname, ftntag);
    IsDoing((char *)"Scan %s", grpname);
    sprintf(temp, "GROUP %s\r\n", grpname);
    nntp_send(temp);
    resp = nntp_receive();
    retval = atoi(strtok(resp, " "));
    if (retval == 480) {
	/*
	 * We must login
	 */
	if (nntp_auth() == FALSE) {
	    WriteError("Authorisation failure");
	    nntp_close();
	    return RETVAL_NOAUTH;
	}
	nntp_send(temp);
	resp = nntp_receive();
	retval = atoi(strtok(resp, " "));
    }
    if (retval != 211) {
	if (retval == 411) {
	    WriteError("No such newsgroup: %s", grpname);
	    return RETVAL_UNEXPECTEDANS;
	}
	WriteError("Unknown response %d to GROUP command", retval);
	return RETVAL_ERROR;
    }

    total = atol(strtok(NULL, " "));
    start = atol(strtok(NULL, " "));
    end   = atol(strtok(NULL, " '\0'"));
    Syslog('m', "GROUP total %d, start %d, end %d, max %d", total, start, end, maxarticles);
    if ((maxarticles) && (total > maxarticles)) {
	start = end - maxarticles;
	total = maxarticles;
	Syslog('m', "NEW:  total %d, start %d, end %d", total, start, end);
    }
    if (!total) {
	return RETVAL_NOARTICLES;
    }

    retval = get_xover(grpname, start, end, art);
    if (retval != RETVAL_OK) {
	tidy_artlist(art);
	return retval;
    }

    if (!do_learn) {
	for (tmp = *art; tmp; tmp = tmp->next) {
	    if (!tmp->isdupe) {
		/*
		 *  If the message isn't a dupe, it must be new for us.
		 */
		get_article(tmp->msgid, ftntag);
		fetched++;
	    }
	}
    }

    tidy_artlist(art);

    if ((maxarticles) && (fetched == maxarticles))
	Syslog('!', "Warning: the max. articles value in newsgroup %s might be to low", grpname);

    Syslog('+', "Fetched %d message%s from %s", fetched, (fetched == 1) ? "":"s", grpname);
    return RETVAL_OK;
}



int get_article(char *msgid, char *ftntag)
{
    char    cmd[81], *resp;
    int	    retval, done = FALSE;
    FILE    *fp = NULL, *dp;
    char    dpath[PATH_MAX];

    Syslog('m', "Get article %s, %s", msgid, ftntag);
    if (!SearchMsgs(ftntag)) {
	WriteError("Search message area %s failed", ftntag);
	return RETVAL_ERROR;
    }

    sprintf(dpath, "%s/tmp/scannews.last", getenv("MBSE_ROOT"));
    dp = fopen(dpath, "w");

    IsDoing("Article %d", (news_in + 1));
    sprintf(cmd, "ARTICLE %s\r\n", msgid);
    fprintf(dp, "ARTICLE %s\n", msgid);
    nntp_send(cmd);
    resp = nntp_receive();
    fprintf(dp, "%s\n", resp);
    retval = atoi(strtok(resp, " "));
    switch (retval) {
	case 412:   WriteError("No newsgroup selected");
		    return RETVAL_UNEXPECTEDANS;
	case 420:   WriteError("No current article has been selected");
		    return RETVAL_UNEXPECTEDANS;
	case 423:   WriteError("No such article in this group");
		    return RETVAL_UNEXPECTEDANS;
	case 430:   WriteError("No such article found");
		    return RETVAL_UNEXPECTEDANS;
	case 220:   if ((fp = tmpfile()) == NULL) {
			WriteError("$Can't open tmpfile");
			return RETVAL_UNEXPECTEDANS;
		    }
		    while (done == FALSE) {
			resp = nntp_receive();
			fwrite(resp, strlen(resp), 1, dp);
			fprintf(dp, "\n");
			fflush(dp);
			if ((strlen(resp) == 1) && (strcmp(resp, ".") == 0)) {
			    done = TRUE;
			} else {
			    fwrite(resp, strlen(resp), 1, fp);
			    fputc('\n', fp);
			}
		    }
		    break;
    }

    IsDoing("Article %d", (news_in));
    retval = rfc2ftn(fp, NULL);
    fclose(fp);
    fclose(dp);
    return retval;
}



int get_xover(char *grpname, long startnr, long endnr, List **art)
{
    char	    cmd[81], *ptr, *ptr2, *resp, *p;
    int		    retval, dupe, done = FALSE;
    long	    nr;
    unsigned long   crc;
    POverview	    pov;

    sprintf(cmd, "XOVER %ld-%ld\r\n", startnr, endnr);
    if ((retval = nntp_cmd(cmd, 224))) {
	switch (retval) {
	    case 412:	WriteError("No newsgroup selected");
			return RETVAL_NOXOVER;
	    case 502:	WriteError("Permission denied");
			return RETVAL_NOXOVER;
	    case 420:	Syslog('m', "No articles in group %s", grpname);
			return RETVAL_OK;
	}
    }

    while (done == FALSE) {
	resp = nntp_receive();
	if ((strlen(resp) == 1) && (strcmp(resp, ".") == 0)) {
	    done = TRUE;
	} else {
	    Marker();
	    Nopper();
	    pov = xoverview;
	    ptr = resp;
	    ptr2 = ptr;

	    /*
	     * First item is the message number.
	     */
	    while (*ptr2 != '\0' && *ptr2 != '\t')
		ptr2++;
	    if (*ptr2 != '\0')
		*(ptr2) = '\0';
	    nr = atol(ptr);
	    ptr = ptr2;
	    ptr++;

	    /*
	     * Search the message-id
	     */
	    while (*ptr != '\0' && pov != NULL && strcmp(pov->header, "Message-ID:") != 0) {
		/*
		 * goto the next field, past the tab.
		 */
		pov = pov->next;

		while (*ptr != '\t' && *ptr != '\0')
		    ptr++;
		if (*ptr != '\0')
		    ptr++;
	    }
	    if (*ptr != '\0' && pov != NULL) {
		/*
		 * Found it, now find start of msgid
		 */
		while (*ptr != '\0' && *ptr != '<')
		    ptr++;
		if(ptr != '\0') {
		    ptr2 = ptr;
		    while(*ptr2 != '\0' && *ptr2 != '>')
			ptr2++;
		    if (*ptr2 != '\0') {
			*(ptr2+1) = '\0';
			p = xstrcpy(ptr);
			p = xstrcat(p, grpname);
			crc = str_crc32(p);
			dupe = CheckDupe(crc, D_NEWS, CFG.nntpdupes);
			fill_artlist(art, ptr, nr, dupe);
			free(p);
			if (CFG.slow_util && do_quiet)
			    msleep(1);
		    }
		}
	    }
	}
    }

    return RETVAL_OK;
}



int get_xoverview(void)
{
    int		retval, len, full, done = FALSE;
    char	*resp;
    POverview	tmp, curptr = NULL;

    Syslog('m', "Getting overview format list");
    if ((retval = nntp_cmd((char *)"LIST overview.fmt\r\n", 215)) == 0) {
	while (done == FALSE) {
	    resp = nntp_receive();
	    if ((strcmp(resp, ".") == 0) && (strlen(resp) == 1)) {
		done = TRUE;
	    } else {
		len = strlen(resp);
		/*
		 * Check for the full flag, which means the field name
		 * is in the xover string.
		 */
		full = (strstr(resp, ":full") == NULL) ? FALSE : TRUE;
		/*
		 * Now get rid of everything back to :
		 */
		while (resp[len] != ':')
		    resp[len--] = '\0';
		len++;

		tmp = malloc(sizeof(Overview));
		tmp->header = calloc(len + 1, sizeof(char));
		strncpy(tmp->header, resp, len);
		tmp->header[len] = '\0';
		tmp->next = NULL;
		tmp->field = NULL;
		tmp->fieldlen = 0;
		tmp->full = full;

		if (curptr == NULL) {
		    /* at head of list */
		    curptr = tmp;
		    xoverview = tmp;
		} else {
		    /* add to linked list */
		    curptr->next = tmp;
		    curptr = tmp;
		}
	    }
	}

//	if ((tmp = xoverview) != NULL) {
//	    Syslog('M', "--Xoverview.fmt list");
//	    while (tmp != NULL) {
//		if (tmp->header != NULL) {
//		    Syslog('M', "item = %s -- full = %s", tmp->header, tmp->full ? "True":"False");
//		}
//		tmp = tmp->next;
//	    }
//	}
    } else {
	return 1;
    }
    return 0;
}