path: root/tests/lacwget.c

/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- */

/*  Lac - Library for asynchronous communication
 *  Copyright (C) 2002, 2003  Søren Sandmann (sandmann@daimi.au.dk)
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Library General Public
 *  License as published by the Free Software Foundation; either
 *  version 2 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Library General Public License for more details.
 *
 *  You should have received a copy of the GNU Library General Public
 *  License along with this library; if not, write to the 
 *  Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 *  Boston, MA  02111-1307, USA.
 */
#include <lac.h>
#include "htmlparser.h"
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>

// #define g_print(...)

static GMainLoop *main_loop;
static GTimer *total = NULL;

static gboolean do_download (LacUri *base_uri, const gchar *uri_str, gboolean try_parse);

typedef struct Data {
    gchar *uri_str;
    gdouble time;
    gdouble first_connect;
    gdouble first_content;
    gdouble first_image_tag;
    gdouble first_idle;
    gdouble first_parser_callback;
    guint   n_content_chunks;
    guint   n_parser_calls;
    gdouble parser_eof;
    gint    n_images;
    gint    n_links;
    gboolean try_parse;
} Data;

static GQueue *statistics = NULL;

typedef struct PageInfo {
    GTimer *timer;
    gchar *main_uri;
    LacUri *base_uri;
    gdouble first_connect;
    gdouble first_image_tag;
    gdouble first_content;
    gdouble first_idle;
    gdouble first_parser_callback;
    GString *unparsed;
    gboolean eof;
    HtmlParser *parser;
    gint idle_id;
    gint n_remaining_images;
    gint n_images;
    gint n_links;
    gboolean done;
    guint n_content_chunks;
    guint n_parser_calls;
    gdouble parser_eof;
    gboolean is_redirect;
    gboolean try_parse;
    FILE *file;
} PageInfo;

static GPtrArray *outstanding_page_infos;

typedef struct ImageInfo {
    FILE *f;
    PageInfo *page_info;
     gchar *uri_str;
} ImageInfo;

static gboolean
print_outstanding (gpointer data)
{
    int i;

    g_print ("outstanding:\n");
    for (i = 0; i < outstanding_page_infos->len; ++i)
    {
        PageInfo *info = outstanding_page_infos->pdata[i];
        if (info->try_parse)
            g_print ("%s: %d images %s\n", info->main_uri, info->n_remaining_images, info->done? "- main page done" : "+ rest of main page");
    }
    return TRUE;
}

static gchar *
truncat (gchar *str)
{
    enum {STRLEN = 200};
    if (strlen (str) > STRLEN)
    {
        gchar *new_str = g_malloc (STRLEN + 1);
        strncpy (new_str, str, STRLEN - 3);
        strncpy (new_str + STRLEN - 3, "...", 3);
        new_str[STRLEN] = '\0';
        return new_str;
    }
    else
        return str;
}

static void
maybe_quit (PageInfo *info)
{
    if (info->n_remaining_images == 0 &&
        ((info->try_parse && info->done && info->eof) ||
         (!info->try_parse && info->eof)))
    {
        Data *data = g_new (Data, 1);
        
        data->uri_str = info->main_uri;
        data->time = g_timer_elapsed (info->timer, NULL);
        data->first_connect = info->first_connect;
        data->first_image_tag = info->first_image_tag;
        data->first_content = info->first_content;
        data->first_idle    = info->first_idle;
        data->first_parser_callback = info->first_parser_callback;
        data->n_content_chunks = info->n_content_chunks;
        data->n_parser_calls = info->n_parser_calls;
        data->parser_eof = info->parser_eof;
        data->n_images = info->n_images;
        data->n_links = info->n_links;
        data->try_parse = info->try_parse;
        
        if (!statistics)
            statistics = g_queue_new ();
        g_queue_push_tail (statistics, data);

        g_ptr_array_remove (outstanding_page_infos, info);
        if (outstanding_page_infos->len == 0)
        {
            int total_requests = 0;
            GList *list;
            
#if 0
            printf ("\n\n\n\n");
#endif
            for (list = statistics->head; list != NULL; list = list->next)
            {
                Data *data = list->data;
                if (data->try_parse)
                {
                    g_print ("%s:\n", data->uri_str);
                    g_print ("    total:                    %f \n", data->time);
                    g_print ("    connected:                %f \n", data->first_connect);
                    g_print ("    first content:            %f \n", data->first_content);
                    g_print ("    first idle:               %f \n", data->first_idle);
                    g_print ("    first parser callback:    %f\n", data->first_parser_callback);
                    g_print ("    first imgtag seen:        %f\n", data->first_image_tag);
                    g_print ("    push eof to parser:       %f\n", data->parser_eof);
                    g_print ("    content chunks:           %d\n", data->n_content_chunks);
                    g_print ("    parser calls:             %d\n", data->n_parser_calls);
                    g_print ("    number of images:         %d\n", data->n_images);
                    g_print ("    number of LINK/href elmts %d\n", data->n_links);
                    g_print ("\n");
                    total_requests += data->n_images + data->n_links + 1;
                }
            }
            printf ("%d requests in ", total_requests);
            printf ("total time: %f\n", g_timer_elapsed (total, NULL));
            g_main_loop_quit (main_loop);
#if 0
            printf ("(press Ctrl-C to quit, or wait 5 seconds\n");
            g_timeout_add (5000, g_main_loop_quit, main_loop);
#endif
        }
    }
}

static gboolean
parse_idle (gpointer data)
{
    PageInfo *page_info = data;

    if (page_info->first_idle == -1.0)
        page_info->first_idle = g_timer_elapsed (page_info->timer, NULL);
    
    if (page_info->unparsed->len > 0)
    {
        page_info->n_parser_calls++;
#if 0
        g_print ("%f: push data (%d bytes)\n",
                 g_timer_elapsed (total, NULL), page_info->unparsed->len);
#endif
        if (page_info->try_parse)
            html_parser_push_data (
                page_info->parser, page_info->unparsed->str, page_info->unparsed->len);
        g_string_truncate (page_info->unparsed, 0);
    }
    
    if (page_info->eof)
    {
        if (page_info->parser)
        {
            g_print ("%f: push eof\n",
                     g_timer_elapsed (total, NULL));
            html_parser_push_eof (page_info->parser);
        }
        page_info->parser_eof = g_timer_elapsed (page_info->timer, NULL);
        page_info->done = TRUE;
    }

    page_info->idle_id = 0;
    maybe_quit (page_info);
    
    return FALSE;
}

static void
get_image (PageInfo *page_info, gchar *image_uri_str)
{
    if (do_download (page_info->base_uri, image_uri_str, FALSE))
        ++page_info->n_images;
}

static void
handle_img_element (PageInfo *page_info, const HtmlParserBeginElementEvent *event)
{
    int i;
    
    if (page_info->first_image_tag == -1.0)
    {
        page_info->first_image_tag =
            g_timer_elapsed (page_info->timer, NULL);
    }

    for (i = 0; i < event->attributes->len; ++i)
    {
        HtmlAttribute *attr =
            event->attributes->pdata[i];
        
        if (g_ascii_strcasecmp (attr->name, "src") == 0)
        {
            if (attr->content)
                get_image (page_info, g_strdup (attr->content));
        }
    }
}

static void
handle_link_element (PageInfo *page_info, const HtmlParserBeginElementEvent *event)
{
    int i;
    gchar *uri = NULL;
    
    for (i = 0; i < event->attributes->len; ++i)
    {
        HtmlAttribute *attr = event->attributes->pdata[i];
        
        if (g_ascii_strcasecmp (attr->name, "href") == 0)
        {
            uri = g_strdup (attr->content);
            break;
        }
    }
    
    if (uri)
    {
        if (do_download (page_info->base_uri, uri, FALSE))
            page_info->n_links++;
    }
}

static void
handle_base_element (PageInfo *page_info, const HtmlParserBeginElementEvent *event)
{
    int i;

    gchar *uri = NULL;
    
    for (i = 0; i < event->attributes->len; ++i)
    {
        HtmlAttribute *attr = event->attributes->pdata[i];

        if (g_ascii_strcasecmp (attr->name, "href") == 0)
        {
            uri = g_strdup (attr->content);
            break;
        }
    }
    
    if (uri)
    {
        LacUri *new_base = lac_uri_new_from_string (page_info->base_uri, uri);
        if (new_base)
        {
            printf ("new base: %s\n", lac_uri_string (new_base));
            page_info->base_uri = new_base;
        }
    }
}

static void
handle_frame_element (PageInfo *page_info, const HtmlParserBeginElementEvent *event)
{
    int i;

    gchar *uri = NULL;
    
    for (i = 0; i < event->attributes->len; ++i)
    {
        HtmlAttribute *attr = event->attributes->pdata[i];

        if (g_ascii_strcasecmp (attr->name, "src") == 0)
        {
            uri = g_strdup (attr->content);
            break;
        }
    }
    
    if (uri)
    {
        do_download (page_info->base_uri, uri, TRUE);
    }
}


static void
handle_meta_element (PageInfo *page_info, const HtmlParserBeginElementEvent *event)
{
    int i;

    gchar *uri = NULL;
    gboolean is_refresh = FALSE;
    
    for (i = 0; i < event->attributes->len; ++i)
    {
        HtmlAttribute *attr = event->attributes->pdata[i];

        if (g_ascii_strcasecmp (attr->name, "http-equiv") == 0 &&
            g_ascii_strcasecmp (attr->content, "refresh") == 0)
        {
            is_refresh = TRUE;
            break;
        }
    }

    if (!is_refresh)
        return;

    for (i = 0; i < event->attributes->len; ++i)
    {
        HtmlAttribute *attr = event->attributes->pdata[i];

        if (g_ascii_strcasecmp (attr->name, "content") == 0)
        {
            char *lower_case = g_ascii_strdown (attr->content, -1);
            const char *pos = strstr (lower_case, ";url=");

            if (pos)
            {
                uri = g_strdup (attr->content + (pos - lower_case) + 5);
            }

            g_free (lower_case);
            break;
        }
    }

    if (uri)
    {
        do_download (page_info->base_uri, uri, TRUE);
    }
}

static void
parser_callback (HtmlParser *parser, const HtmlParserEvent *event)
{
    PageInfo *page_info = html_parser_get_data (parser);
    if (page_info->first_parser_callback == -1.0)
        page_info->first_parser_callback = g_timer_elapsed (page_info->timer, NULL);

    switch (event->type)
    {
    case HTML_PARSER_BEGIN_ELEMENT:
#if 0
        g_print ("%f: begin %s\n",
                 g_timer_elapsed (total, NULL), event->begin_element.name);
#endif
        if (g_ascii_strcasecmp (event->begin_element.name, "img") == 0)
        {
            handle_img_element (page_info, &(event->begin_element));
        }
        else if (g_ascii_strcasecmp (event->begin_element.name, "link") == 0)
        {
            handle_link_element (page_info, &(event->begin_element));
        }
        else if (g_ascii_strcasecmp (event->begin_element.name, "frame") == 0)
        {
            handle_frame_element (page_info, &(event->begin_element));
        }
        else if (g_ascii_strcasecmp (event->begin_element.name, "base") == 0)
        {
            handle_base_element (page_info, &(event->begin_element));
        }
        else if (g_ascii_strcasecmp (event->begin_element.name, "meta") == 0)
        {
            handle_meta_element (page_info, &(event->begin_element));
        }
        break;

    case HTML_PARSER_END_ELEMENT:
#if 0
        g_print ("%f: end %s\n",
                 g_timer_elapsed (total, NULL), event->end_element.name);
#endif
        break;

    case HTML_PARSER_COMMENT:
#if 0
        g_print ("%f: comment\n", g_timer_elapsed (total, NULL));
#endif
#if 0
        g_print ("%f: comment: %s\n",
                 g_timer_elapsed (total, NULL), event->comment.text);
#endif
        break;
    case HTML_PARSER_END_DOCUMENT:
	g_print ("END DOCUMENT REPORTED BY PARSER\n");
	break;
    }
}

static void
queue_parse (PageInfo *info)
{
    if (!info->idle_id)
        info->idle_id = g_idle_add (parse_idle, info);
}

static void
http_callback (LacHttpRequest *request, const LacHttpEvent *event)
{
    PageInfo *page_info = lac_http_request_get_data (request);
    
    switch (event->type)
    {
    case LAC_HTTP_EVENT_HOST_FOUND:
        g_print ("%s: dns lookup complete at %f (%s: %s)\n",
                 page_info->main_uri,
                 g_timer_elapsed (total, NULL),
                 event->host_found.hostname,
                 lac_address_to_string (event->host_found.address));
	break;
	
    case LAC_HTTP_EVENT_SENT:
        page_info->first_connect = g_timer_elapsed (page_info->timer, NULL);
	g_print ("%s: sent \n", page_info->main_uri);
	break;
	
    case LAC_HTTP_EVENT_CONNECTING:
	break;
	
    case LAC_HTTP_EVENT_STATUS_LINE:
	g_print ("%s:  status line: HTTP/%d.%d %d %s\n",
                 truncat (page_info->main_uri),
                 event->status_line.major,
                 event->status_line.minor,
		 event->status_line.status_code,
                 event->status_line.reason_phrase);
        if (event->status_line.status_code == 302 ||
            event->status_line.status_code == 301)
        {
            g_print ("IS REDIRECT!\n");
            page_info->is_redirect = TRUE;
        }
	break;

    case LAC_HTTP_EVENT_NO_STATUS_LINE:
        g_print ("%s: no status line (HTTP/0.9 response)\n",
                 truncat (page_info->main_uri));
        break;
	
    case LAC_HTTP_EVENT_HEADER:
        g_print ("%s: header  (%s: %s)\n", page_info->main_uri, event->header.header, event->header.value);
        if (page_info->is_redirect && g_ascii_strcasecmp (event->header.header, "Location") == 0)
        {
            do_download (page_info->base_uri, event->header.value, page_info->try_parse);
        }
	break;
	
    case LAC_HTTP_EVENT_BEGIN_CONTENT:
#if 0
	g_print ("%s:   begin content (length %d)\n", page_info->main_uri, event->begin_content.length);
#endif
	break;
	
    case LAC_HTTP_EVENT_CONTENT:
#if 0
 {
     int i;
     for (i = 0; i < event->content.length; ++i)
         printf ("%c", event->content.data[i]);
 }
#endif
#if 0
	g_print ("%s:   content chunk (size: %d)\n", page_info->main_uri, event->content.length);
#endif
        if (page_info->file)
            fwrite (event->content.data, 1, event->content.length, page_info->file);
        g_string_append_len (page_info->unparsed, event->content.data, event->content.length);
        if (page_info->first_content == -1.0)
            page_info->first_content = g_timer_elapsed (page_info->timer, NULL);
        
        queue_parse (page_info);
#if 0
        parse_idle (page_info);
#endif
        page_info->n_content_chunks++;
        break;
     
    case LAC_HTTP_EVENT_END_CONTENT:
#if 0
	printf ("%s:   end_content (total: %d)\n", page_info->main_uri, event->end_content.total);
#endif
        g_print ("%s: DONE (at total: %f, this page: %f\n",
                 truncat (page_info->main_uri),
                 g_timer_elapsed (total, NULL),
                 g_timer_elapsed (page_info->timer, NULL));
        page_info->eof = TRUE;
        queue_parse (page_info);
        
#if 0
        parse_idle (page_info);
#endif
	break;
	
    case LAC_HTTP_EVENT_ERROR:
	printf ("%s:  error (%s)\n", truncat (page_info->main_uri), event->error.err->message);
        page_info->eof = TRUE;
        queue_parse (page_info);
	break;
	
    default:
	g_warning ("%s:  UNKNOWN (%d)\n", truncat (page_info->main_uri), event->type);
	break;
    }
}

static gboolean
already_downloaded (const LacUri *uri)
{
    static GPtrArray *already;
    int i;

    if (!already)
        already = g_ptr_array_new ();

    for (i = 0; i < already->len; ++i)
    {
        if (lac_uri_equal (already->pdata[i], uri))
            return TRUE;
    }

    g_ptr_array_add (already, lac_uri_copy (uri));
    return FALSE;
}

static char *
create_name (const char *uri_str)
{
    char *last_slash = strrchr (uri_str, '/');

    if (last_slash)
        return g_strdup (last_slash + 1);
    else
        return g_strdup (uri_str);
}

static gboolean
do_download (LacUri *base_uri, const gchar *uri_str, gboolean try_parse)
{
    LacHttpRequest *request;
    PageInfo *page_info = g_new (PageInfo, 1);
    LacUri *uri;
    page_info->timer = g_timer_new ();
    page_info->try_parse = try_parse;

    page_info->file = fopen (create_name (uri_str), "w");

    g_print ("write: %s\n", create_name (uri_str));
    
#if 0
    printf ("uri: %s\n", uri_str);
#endif
    
    if (page_info->try_parse)
        g_print ("DOWNLAOD TRUE %s ********\n", uri_str);

    uri = lac_uri_new_from_string (base_uri, uri_str);
    if (!uri)
    {
        printf ("bad uri: %s\n", uri_str);
        return FALSE ;
    }
    else if (uri->scheme != LAC_SCHEME_HTTP)
    {
        printf ("not an http uri: %s\n", uri_str);
        return FALSE;
    }
    if (already_downloaded (uri))
        return FALSE;

#if 0
    printf ("download: %s\n", uri_str);
#endif
    
    request = lac_http_request_new_get (
        uri, http_callback, page_info);
    
    page_info->main_uri = g_strdup (uri_str);
    page_info->base_uri = uri;
    if (page_info->try_parse)
        page_info->parser = html_parser_new (parser_callback, page_info);
    else
        page_info->parser = NULL;
    page_info->n_remaining_images = 0;
    page_info->n_images = 0;
    page_info->n_links = 0;
    page_info->done = FALSE;
    page_info->idle_id = 0;
    page_info->eof = FALSE;
    page_info->unparsed = g_string_new ("");
    page_info->first_connect = -1.0;
    page_info->first_content = -1.0;
    page_info->first_image_tag = -1.0;
    page_info->first_idle = -1.0;
    page_info->first_parser_callback = -1.0;
    page_info->parser_eof = -1.0;
    page_info->n_content_chunks = 0;
    page_info->n_parser_calls = 0;
    page_info->is_redirect = FALSE;
    
    g_ptr_array_add (outstanding_page_infos, page_info);

#if 0
    printf ("length: %d\n", outstanding_page_infos->len);
#endif
    
    g_print ("dispatched %s at %f\n", page_info->main_uri, g_timer_elapsed (total, NULL));
    lac_http_request_dispatch (request);
    return TRUE;
}

int
main (int argc, char *argv[])
{
    int i;
    
    g_log_set_always_fatal (G_LOG_LEVEL_WARNING | G_LOG_LEVEL_ERROR |
                            G_LOG_LEVEL_CRITICAL);
    
    
    lac_set_verbose (TRUE);

    outstanding_page_infos = g_ptr_array_new ();
    total = g_timer_new ();
    if (argc > 1)
    {
	for (i = 1; i < argc; ++i)
            do_download (NULL, argv[i], TRUE);
    }
    else
    {
	printf ("usage %s <uris>\n", argv[0]);
	return 1;
    }
    
    main_loop = g_main_loop_new (NULL, TRUE);
#if 0
    g_timeout_add (5000, (GSourceFunc) print_outstanding, NULL);
    g_timeout_add (38000, (GSourceFunc) g_main_loop_quit, main_loop);
#endif
    g_print ("dispatched requests\n");
    g_assert (main_loop);
    if (outstanding_page_infos->len > 0)
        g_main_loop_run (main_loop);
    
    return g_timer_elapsed (total, NULL) * 10000;
}