fetch.c

Go to the documentation of this file.

/*
 *  $Id: fetch.c,v 1.19 2006/04/17 22:45:19 ghost666 Exp $
 *
 *  rssbgr - RSS backgroud reader
 *  Copyright (C) 2005, 2006 Piotr 'GhosT' Wydrych
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

#include "fetch.h"

int find_in_file(FILE *fd, char *string)
{
 char line[32];

 DEBUG3("find_in_file: %p, %s", fd, string);
 if (fd == NULL) return 0;
 rewind(fd);
 while(!feof(fd))
 {
  fscanf(fd, "%s\n", line);
  if (!strcmp(line, string))
   return 1;
 }
 return 0;
}

void process_rss(xmlNode *root_node, int config_item_i)
{
 xmlNode *cur_node, *cur_sub_node, *cur_sub_sub_node;
 char *channel_link = NULL, *channel_title = NULL, *channel_desc = NULL; /* NULL the pointers, because we won't malloc */
 char *item_link = NULL, *item_title = NULL, *item_desc = NULL;
 char *filename, *filename_new;
 char *digest = (char *)malloc(33);
 char *command;
 FILE *fd, *fd_new;
 
 filename = (char *)malloc(PATH_MAX);
 md5(digest, config[config_item_i].url);
 sprintf(filename, "%s/%s", directory, digest);
 filename_new = (char *)malloc(PATH_MAX);
 sprintf(filename_new, "%s.new", filename);
 fd = fopen(filename, "r"); /* don't bother checkig if not NULL, do it in find_in_file() */
 fd_new = fopen(filename_new, "w");
 if (!strcmp(root_node->name, "rss"))
  for (cur_node = root_node->children; cur_node != NULL; cur_node = cur_node->next)
   if (!strcmp(cur_node->name, "channel"))
    for (cur_sub_node = cur_node->children; cur_sub_node != NULL; cur_sub_node = cur_sub_node->next)
    {
     if (!strcmp(cur_sub_node->name, "item"))
     {
      for (cur_sub_sub_node = cur_sub_node->children; cur_sub_sub_node != NULL; cur_sub_sub_node = cur_sub_sub_node->next)
      {
       if (!strcmp(cur_sub_sub_node->name, "link"))
        item_link = xmlNodeListGetString(root_node->doc, cur_sub_sub_node->xmlChildrenNode, 1);
       else if (!strcmp(cur_sub_sub_node->name, "title") && strstr(config[config_item_i].action, "{t}"))
        item_title = xmlNodeListGetString(root_node->doc, cur_sub_sub_node->xmlChildrenNode, 1);
       else if (!strcmp(cur_sub_sub_node->name, "description") && strstr(config[config_item_i].action, "{d}"))
        item_desc = xmlNodeListGetString(root_node->doc, cur_sub_sub_node->xmlChildrenNode, 1);
        }
      md5(digest, item_link);
      if (!find_in_file(fd, digest))
      {
       command=replace_fields(config[config_item_i].action, channel_link, channel_title, channel_desc, item_link, item_title, item_desc);
       execute(command);
       free(command);
      }
      if (fd_new != NULL) fprintf(fd_new, "%s\n", digest);
      free(item_link); free(item_title); free(item_desc); item_link = item_title = item_desc = NULL;
     }
     else if (!strcmp(cur_sub_node->name, "link") && strstr(config[config_item_i].action, "{L}"))
      channel_link = xmlNodeListGetString(root_node->doc, cur_sub_node->xmlChildrenNode, 1);
     else if (!strcmp(cur_sub_node->name, "title") && strstr(config[config_item_i].action, "{T}"))
      channel_title = xmlNodeListGetString(root_node->doc, cur_sub_node->xmlChildrenNode, 1);
     else if (!strcmp(cur_sub_node->name, "description") && strstr(config[config_item_i].action, "{D}"))
      channel_desc = xmlNodeListGetString(root_node->doc, cur_sub_node->xmlChildrenNode, 1);
    }
 if (fd != NULL) fclose(fd);
 if (fd_new != NULL)
 {
  fclose(fd_new);
  rename(filename_new, filename);
 }
 free(channel_link); free(channel_title); free(channel_title);
 free(filename); free(filename_new); free(digest);
}

time_t string_to_time(char *data)
{
 struct tm tm;
 if (strptime(data, "%a, %d %b %Y %H:%M:%S", &tm) == NULL) return -1; 
 tm.tm_isdst = -1;
 return mktime(&tm);
}

enum HTTP_STATE process_header(char *data, int len, int config_item_i)
{
 size_t pos, _len, cookie_pos;
 int cookie_i, i;
 time_t server_time = time(NULL), time_diff = 0;  /* time_diff = localtime - servertime */
 char *url, *host, *request;
 uint16_t port;

 if (len < 16) return HTTP_STATE_ERROR; /* 16 = strlen("HTTP/1.x xxx") */
 if (strncmp("HTTP/1.", data, 7)) return HTTP_STATE_ERROR; /* are there any HTTP/0.9 servers? ;-) */
 pos = 9;
 switch(data[pos])
 {
  case '2':
   if (data[pos+2] == '4') return HTTP_STATE_ERROR; /* HTTP/1.0 204 No Content */
   while ((pos = (size_t)strchr(data+pos, '\n')) && (pos = pos-(size_t)data+1) <= len)
   {
    if (!strncmp("Date:", data+pos, 5))
    {
     server_time = ((server_time = string_to_time(data+pos+5)) != -1)?(server_time):(time(NULL));
     time_diff = time(NULL) - server_time;
    }
    else if (!strncmp("Set-Cookie:", data+pos, 11)) 
    {
     if (strchr(data+pos+11, '\r') && strchr(data+pos+11, '\n'))
      _len = MIN(strchr(data+pos+11, ';'), MIN(strchr(data+pos+11, '\r'), strchr(data+pos+11, '\n')))-(data+pos+11);
     else if (strchr(data+pos+11, '\r'))
      _len = MIN(strchr(data+pos+11, ';'), strchr(data+pos+11, '\r'))-(data+pos+11);
     else
      _len = MIN(strchr(data+pos+11, ';'), strchr(data+pos+11, '\n'))-(data+pos+11);
     cookie_i = -1;
     for (i = 0; i < config[config_item_i].cookies_count; ++i)
      if (!strncmp(data+pos+11, config[config_item_i].cookies[i].data, strchr(data+pos+11, '=')-(data+pos+11)))
       cookie_i = i;
     if (cookie_i == -1)
     {
      cookie_i = config[config_item_i].cookies_count++;
      config[config_item_i].cookies = (struct cookie *)realloc(config[config_item_i].cookies, config[config_item_i].cookies_count*sizeof(struct cookie));
     }
     else
      free(config[config_item_i].cookies[cookie_i].data);
     config[config_item_i].cookies[cookie_i].data = (char *)malloc(_len+1);
     strlcpy(config[config_item_i].cookies[cookie_i].data, data+pos+11, _len+1);
     cookie_pos = ((cookie_pos = (size_t)strstr(data+pos+11, "expires="))?(cookie_pos-(size_t)data):(0));
     if (cookie_pos && cookie_pos < (size_t)strchr(data+pos, '\n'))
     {
      if ((config[config_item_i].cookies[cookie_i].expires = string_to_time(data+cookie_pos+8)) != -1)
       config[config_item_i].cookies[cookie_i].expires += time_diff;
     }
     else
      config[config_item_i].cookies[cookie_i].expires = -1;
     DEBUG4("process_header: %d cookie: %s, expires: %d", cookie_i, config[config_item_i].cookies[cookie_i].data, (int)config[config_item_i].cookies[cookie_i].expires)
    }
   }
   return HTTP_STATE_OK;
  case '3':
   while ((pos = (size_t)strchr(data+pos, '\n')) && (pos = pos-(size_t)data+1) <= len)
   {
    if (!strncmp("Location:", data+pos, 9))
    {
     _len = MIN(strchr(data+pos+9, '\r'), strchr(data+pos+9, '\n'))-(data+pos+9);
     pos+=9; while (data[pos] == ' ') ++pos;
     _len = MIN(strchr(data+pos, '\r'), strchr(data+pos, '\n'))-(data+pos);
     url = (char *)malloc(_len+1);
     strlcpy(url, data+pos, _len+1);
     DEBUG2("process_header: Redirect: %s", url)
     if (parse_url(url, &host, &port, &request) == -1)
     {
      free(url); free(host); free(request); return HTTP_STATE_ERROR;
     }
     if (strcmp(host, config[config_item_i].host))
     {
      for (; config[config_item_i].cookies_count; --config[config_item_i].cookies_count)
       free(config[config_item_i].cookies[config[config_item_i].cookies_count-1].data);
      free(config[config_item_i].cookies);
     }
     config[config_item_i].host = realloc(config[config_item_i].host, strlen(host)+1);
     strcpy(config[config_item_i].host, host);
     config[config_item_i].port = port;
     config[config_item_i].request = realloc(config[config_item_i].request, strlen(request)+1);
     strcpy(config[config_item_i].request, request);
     free(url); free(host); free(request);
     return HTTP_STATE_REDIRECT;
    }
   }
  default:
   return HTTP_STATE_ERROR;
 }
}

void *fetch_thread(void *parameters)
{
 char *content, *header, *request;
 int contentsize, i, len;
 char *pos_rn, *pos_n, *pos_c; /* positions of "\r\n\r\n", "\n\n", content */
 xmlDoc *xml_doc;
 xmlNode *xml_node;

fetch_thread_loop:
 request = (char *)malloc(strlen(config[(size_t)parameters].request)+1);
 strcpy(request, config[(size_t)parameters].request);
 len = 0;
 for (i = 0; i<config[(size_t)parameters].cookies_count; ++i)
  if (config[(size_t)parameters].cookies[i].expires == -1 || config[(size_t)parameters].cookies[i].expires >= time(NULL))
  {
   if (!len)
   {
    len = strlen(config[(size_t)parameters].cookies[i].data)+11;
    request = (char *)realloc(request, strlen(config[(size_t)parameters].request)+len);
    strcat(request, "Cookie: ");
    strcat(request, config[(size_t)parameters].cookies[i].data);
   }
   else
   {
    len += strlen(config[(size_t)parameters].cookies[i].data)+2;
    request = (char *)realloc(request, strlen(config[(size_t)parameters].request)+len);
    strcat(request, "; ");
    strcat(request, config[(size_t)parameters].cookies[i].data);
   }
  }
 if (len) strcat(request, "\r\n");
 DEBUG2("Request: %s", request);
 if ((content = fetch(&contentsize, config[(size_t)parameters].host, config[(size_t)parameters].port, request)) == NULL) goto skip;
 free(request);
 pos_rn = strstr(content, "\r\n\r\n");
 pos_n = strstr(content, "\n\n");
 if ((pos_rn != NULL) && (pos_n != NULL))
  pos_c = MIN(pos_rn+4, pos_n+2);
 else
  if ((pos_rn == NULL) && (pos_n == NULL))
   pos_c = content; /* this shouln't happen, but... ;-) */
  else
   if (pos_rn != NULL)
    pos_c = pos_rn+4;
   else
    pos_c = pos_n+2;

 header = (char *)malloc(pos_c-content+1);
 strlcpy(header, content, pos_c-content+1);
 switch (process_header(header, pos_c-content, (size_t)parameters))
 {
  case HTTP_STATE_ERROR: /* error - wait and try again */
   free(header);
   goto skip;
  case HTTP_STATE_REDIRECT: /* redirection - go now to the new location */
   free(content);
   free(header);
   goto fetch_thread_loop;
  case HTTP_STATE_OK:
   free(header);
 }

 LIBXML_TEST_VERSION

 xml_doc = xmlReadMemory(pos_c, content+contentsize-pos_c, NULL, NULL, XML_PARSE_NOERROR | XML_PARSE_NOWARNING);
 if (xml_doc != NULL)
 {
  xml_node = xmlDocGetRootElement(xml_doc);
  process_rss(xml_node, (size_t)parameters);
  xmlFreeDoc(xml_doc);
  xmlCleanupParser();
 }
skip:
 free(content);
 DEBUG2("fetch_thread: sleeping %dsec...", config[(size_t)parameters].freq)
 sleep(config[(size_t)parameters].freq);
 goto fetch_thread_loop;
}

char *fetch(int *responsesize, const char *host, const uint16_t port, const char *request)
{
 DEBUG1("fetch: start")

 int fetch_socket;
 int w, r; /* wrote, read */
 char *buffer = (char *)malloc(FETCH_BUFFER_SIZE);
 char *buffer2 = (char *)malloc(FETCH_BUFFER_SIZE);
 struct sockaddr_in address;
 struct hostent *h;
 
 *responsesize=-1;
 DEBUG1("fetch: opening socket");
 if ((fetch_socket = socket(AF_INET, SOCK_STREAM, 0)) <= 0) return NULL;
 DEBUG1("fetch: socket open")
 address.sin_family = AF_INET;
 address.sin_port = htons(port);
 DEBUG2("fetch: resolving %s", host);
 if ((h = gethostbyname(host)) == NULL) return NULL;
 memcpy(&address.sin_addr, *(h->h_addr_list), sizeof(address.sin_addr));
 if (connect(fetch_socket, (struct sockaddr *)&address, sizeof(address)) != 0) return NULL;
 DEBUG1("fetch: connection accepted")
 w = 0;
 while (w < strlen(request))
 {
  send(fetch_socket, request+w, (WRITE_BUFFER_SIZE < strlen(request+w))?(WRITE_BUFFER_SIZE):(strlen(request+w)), 0);
  w+=WRITE_BUFFER_SIZE;
 }
 send(fetch_socket, "\r\n", 2, 0);
 *responsesize=0;
 while ((r = read(fetch_socket, buffer, FETCH_BUFFER_SIZE)) > 0)
 {
  *responsesize+=r;
  buffer2=(char *)realloc(buffer2, (*responsesize)+1);
  strncat(buffer2, buffer, r);
 }
 close(fetch_socket);
 free(buffer);

 DEBUG1("fetch: end")
 return buffer2;
}

---