diff -r 3646f7f4fcfc -r 5036deb1fcaf .hgignore
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.hgignore	Thu Jul 24 16:18:29 2008 +0300
@@ -0,0 +1,8 @@
+syntax: regexp
+
+^objs/
+^bin(-server)?/
+^Makefile(.am)?$
+\.swp$
+^src/rev\.cpp$
+^config\.(log|cache|cache\.source\.list)$
diff -r 3646f7f4fcfc -r 5036deb1fcaf config.lib
--- a/config.lib	Fri Jul 18 12:20:30 2008 +0000
+++ b/config.lib	Thu Jul 24 16:18:29 2008 +0300
@@ -52,6 +52,7 @@
 	with_cocoa="1"
 	with_zlib="1"
 	with_png="1"
+	with_curl="1"
 	with_makedepend="1"
 	with_direct_music="1"
 	with_sort="1"
@@ -109,6 +110,7 @@
 		with_cocoa
 		with_zlib
 		with_png
+		with_curl
 		with_makedepend
 		with_direct_music
 		with_sort
@@ -261,6 +263,13 @@
 			--with-libpng)                with_png="2";;
 			--without-libpng)             with_png="0";;
 			--with-libpng=*)              with_png="$optarg";;
+
+            --with-curl)                  with_curl="2";;
+			--without-curl)               with_curl="0";;
+			--with-curl=*)                with_curl="$optarg";;
+			--with-libcurl)               with_curl="2";;
+			--without-libcurl)            with_curl="0";;
+			--with-libcurl=*)             with_curl="$optarg";;
 
 			--with-libtimidity)           with_libtimidity="2";;
 			--without-libtimidity)        with_libtimidity="0";;
@@ -562,6 +571,7 @@
 
 	detect_zlib
 	detect_png
+	detect_curl
 	detect_freetype
 	detect_fontconfig
 	detect_iconv
@@ -1083,6 +1093,14 @@
 		else
 			LIBS="$LIBS `$png_config --ldflags | tr '\n\r' '  '`"
 		fi
+    fi
+
+    if [ -n "$curl_config" ]; then
+		CFLAGS="$CFLAGS -DWITH_CURL"
+		CFLAGS="$CFLAGS `$curl_config --cflags  | tr '\n\r' '  '`"
+        
+        # XXX: does this break on enable_static? curl-config doesn't take a --static option
+        LIBS="$LIBS `$curl_config --libs | tr '\n\r' '  '`"
 	fi
 
 	if [ -n "$fontconfig_config" ]; then
@@ -1843,6 +1861,45 @@
 	log 1 "checking libpng... found"
 }
 
+detect_curl() {
+	# 0 means no, 1 is auto-detect, 2 is force
+	if [ "$with_curl" = "0" ]; then
+		log 1 "checking curl... disabled"
+
+		curl_config=""
+		return 0
+	fi
+
+	if [ "$with_curl" = "1" ] || [ "$with_curl" = "" ] || [ "$with_curl" = "2" ]; then
+		curl_config="curl-config"
+	else
+		curl_config="$with_curl"
+	fi
+
+	version=`$curl_config --version 2>/dev/null`
+	ret=$?
+	log 2 "executing $curl_config --version"
+	log 2 "  returned $version"
+	log 2 "  exit code $ret"
+
+	if [ -z "$version" ] || [ "$ret" != "0" ]; then
+		log 1 "checking libcurl... not found"
+
+		# It was forced, so it should be found.
+		if [ "$with_curl" != "1" ]; then
+			log 1 "configure: error: curl-config couldn't be found"
+			log 1 "configure: error: you supplied '$with_curl', but it seems invalid"
+			exit 1
+		fi
+
+		curl_config=""
+		return 0
+	fi
+
+	log 1 "checking libcurl... found"
+
+}
+
 detect_freetype() {
 	# 0 means no, 1 is auto-detect, 2 is force
 	if [ "$with_freetype" = "0" ]; then
@@ -2378,6 +2435,7 @@
 	echo "  --with-sdl[=sdl-config]        enables SDL video driver support"
 	echo "  --with-zlib[=zlib.a]           enables zlib support"
 	echo "  --with-png[=libpng-config]     enables libpng support"
+	echo "  --with-curl[=curl-config]      enables libcurl support"
 	echo "  --with-freetype[=freetype-config]"
 	echo "                                 enables libfreetype support"
 	echo "  --with-fontconfig[=pkg-config fontconfig]"
diff -r 3646f7f4fcfc -r 5036deb1fcaf source.list
--- a/source.list	Fri Jul 18 12:20:30 2008 +0000
+++ b/source.list	Thu Jul 24 16:18:29 2008 +0300
@@ -45,6 +45,7 @@
 network/network_gamelist.cpp
 network/network_server.cpp
 network/network_udp.cpp
+network/newgrf_download.cpp
 npf.cpp
 oldloader.cpp
 oldpool.cpp
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/debug.cpp
--- a/src/debug.cpp	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/debug.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -31,6 +31,7 @@
 int _debug_sl_level;
 int _debug_station_level;
 int _debug_gamelog_level;
+int _debug_grfdl_level;
 
 
 struct DebugLevel {
@@ -56,6 +57,7 @@
 	DEBUG_LEVEL(sl),
 	DEBUG_LEVEL(station),
 	DEBUG_LEVEL(gamelog),
+	DEBUG_LEVEL(grfdl),
 	};
 #undef DEBUG_LEVEL
 
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/debug.h
--- a/src/debug.h	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/debug.h	Thu Jul 24 16:18:29 2008 +0300
@@ -48,6 +48,7 @@
 	extern int _debug_sl_level;
 	extern int _debug_station_level;
 	extern int _debug_gamelog_level;
+	extern int _debug_grfdl_level;
 
 	void CDECL debug(const char *dbg, ...);
 #endif /* NO_DEBUG_MESSAGES */
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/fileio.cpp
--- a/src/fileio.cpp	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/fileio.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -216,7 +216,8 @@
 	"scenario" PATHSEP "heightmap" PATHSEP,
 	"gm" PATHSEP,
 	"data" PATHSEP,
-	"lang" PATHSEP
+	"lang" PATHSEP,
+    "cache" PATHSEP
 };
 
 const char *_searchpaths[NUM_SEARCHPATHS];
@@ -482,7 +483,7 @@
 #endif
 }
 
-static bool TarListAddFile(const char *filename)
+static bool TarListAddFile(const char *filename, Subdirectory subdir)
 {
 	/* The TAR-header, repeated for every file */
 	typedef struct TarHeader {
@@ -515,6 +516,7 @@
 
 	TarListEntry *tar_entry = MallocT<TarListEntry>(1);
 	tar_entry->filename = strdup(filename);
+    tar_entry->subdir = subdir;
 	_tar_list.insert(TarList::value_type(filename, tar_entry));
 
 	TarLinkList links; ///< Temporary list to collect links
@@ -693,7 +695,7 @@
 	return true;
 }
 
-static int ScanPathForTarFiles(const char *path, size_t basepath_length)
+static int ScanPathForTarFiles(const char *path, size_t basepath_length, Subdirectory subdir)
 {
 	extern bool FiosIsValidFile(const char *path, const struct dirent *ent, struct stat *sb);
 
@@ -716,7 +718,7 @@
 			/* Directory */
 			if (strcmp(d_name, ".") == 0 || strcmp(d_name, "..") == 0) continue;
 			AppendPathSeparator(filename, lengthof(filename));
-			num += ScanPathForTarFiles(filename, basepath_length);
+			num += ScanPathForTarFiles(filename, basepath_length, subdir);
 		} else if (sb.st_mode & S_IFREG) {
 			/* File */
 			char *ext = strrchr(filename, '.');
@@ -725,7 +727,7 @@
 			if (ext == NULL) continue;
 			if (strcasecmp(ext, ".tar") != 0) continue;
 
-			if (TarListAddFile(filename)) num++;
+			if (TarListAddFile(filename, subdir)) num++;
 		}
 	}
 
@@ -742,7 +744,10 @@
 	DEBUG(misc, 1, "Scanning for tars");
 	FOR_ALL_SEARCHPATHS(sp) {
 		FioAppendDirectory(path, MAX_PATH, sp, DATA_DIR);
-		num += ScanPathForTarFiles(path, strlen(path));
+		num += ScanPathForTarFiles(path, strlen(path), DATA_DIR);
+		
+        FioAppendDirectory(path, MAX_PATH, sp, CACHE_DIR);
+		num += ScanPathForTarFiles(path, strlen(path), CACHE_DIR);
 	}
 	DEBUG(misc, 1, "Scan complete, found %d files", num);
 }
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/fileio.h
--- a/src/fileio.h	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/fileio.h	Thu Jul 24 16:18:29 2008 +0300
@@ -32,6 +32,7 @@
 	GM_DIR,        ///< Subdirectory for all music
 	DATA_DIR,      ///< Subdirectory for all data (GRFs, sample.cat, intro game)
 	LANG_DIR,      ///< Subdirectory for all translation files
+    CACHE_DIR,     ///< Subdirectory for all cache data (not user-visible)
 	NUM_SUBDIRS,   ///< Number of subdirectories
 	NO_DIRECTORY,  ///< A path without any base directory
 };
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/lang/english.txt
--- a/src/lang/english.txt	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/lang/english.txt	Thu Jul 24 16:18:29 2008 +0300
@@ -3144,6 +3144,7 @@
 STR_PERFORMANCE_DETAIL_LOAN_TIP                                 :{BLACK}The amount of money this company has taken on loan
 STR_PERFORMANCE_DETAIL_TOTAL_TIP                                :{BLACK}Total points out of possible points
 
+STR_NEWGRF_DOWNLOAD_BUTTON                                      :{BLACK}NewGRF Download
 STR_NEWGRF_SETTINGS_BUTTON                                      :{BLACK}NewGRF Settings
 STR_NEWGRF_SETTINGS_CAPTION                                     :{WHITE}NewGRF settings
 STR_NEWGRF_APPLY_CHANGES                                        :{BLACK}Apply changes
@@ -3189,6 +3190,14 @@
 STR_NEWGRF_RESCAN_FILES                                         :{BLACK}Rescan files
 STR_NEWGRF_RESCAN_FILES_TIP                                     :{BLACK}Update the list of available NewGRF files
 STR_NEWGRF_DUPLICATE_GRFID                                      :{WHITE}Cannot add file: duplicate GRF ID
+
+STR_NEWGRF_DOWNLOAD_CAPTION                                     :{WHITE}Download NewGRF files
+STR_NEWGRF_DOWNLOAD_LIST_TIP                                    :{BLACK}A list of the NewGRF files that are missing.
+STR_NEWGRF_CHECK_AVAILABLE                                      :{BLACK}Check Available
+STR_NEWGRF_DOWNLOAD_AVAILABLE                                   :{BLACK}Download Available
+STR_NEWGRF_DOWNLOAD_CHECK_FAILED                                :{WHITE}Checking for available NewGRFs failed
+STR_NEWGRF_DOWNLOAD_CHECK_NONE                                  :{WHITE}No NewGRFs available for download
+STR_NEWGRF_DOWNLOAD_FAILED                                      :{WHITE}Downloading the NewGRFs failed
 
 STR_NEWGRF_NOT_FOUND                                            :{RED}Matching file not found
 STR_NEWGRF_DISABLED                                             :{RED}Disabled
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/network/network_gui.cpp
--- a/src/network/network_gui.cpp	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/network/network_gui.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -21,6 +21,7 @@
 #include "network_udp.h"
 #include "../town.h"
 #include "../newgrf.h"
+#include "newgrf_download.h"
 #include "../functions.h"
 #include "../window_func.h"
 #include "../core/alloc_func.hpp"
@@ -107,6 +108,7 @@
 	NGWW_DETAILS,       ///< Panel with game details
 	NGWW_JOIN,          ///< 'Join game' button
 	NGWW_REFRESH,       ///< 'Refresh server' button
+    NGWW_NEWGRF_DL,     ///< 'NewGRF Download' button
 	NGWW_NEWGRF,        ///< 'NewGRF Settings' button
 
 	NGWW_FIND,          ///< 'Find server' button
@@ -288,6 +290,11 @@
 		this->SetWidgetHiddenState(NGWW_NEWGRF, sel == NULL ||
 				!sel->online ||
 				sel->info.grfconfig == NULL);
+        
+        /* 'NewGRF Download' button invisible if no incompatible NewGRFs are used */
+		this->SetWidgetHiddenState(NGWW_NEWGRF_DL, sel == NULL ||
+				!sel->online ||
+				sel->info.compatible || !sel->info.version_compatible);
 
 		SetDParam(0, 0x00);
 		SetDParam(1, _lan_internet_types_dropdown[_settings_client.network.lan_internet]);
@@ -475,6 +482,10 @@
 			case NGWW_NEWGRF: // NewGRF Settings
 				if (this->server != NULL) ShowNewGRFSettings(false, false, false, &this->server->info.grfconfig);
 				break;
+
+            case NGWW_NEWGRF_DL: // NewGRF Download
+				if (this->server != NULL) ShowNewGRFDownload(&this->server->info);
+                break;
 		}
 	}
 
@@ -593,6 +604,7 @@
 { WWT_PUSHTXTBTN,   RESIZE_LRTB,   BTC,   215,   315,   215,   226, STR_NETWORK_JOIN_GAME,            STR_NULL},                         // NGWW_JOIN
 { WWT_PUSHTXTBTN,   RESIZE_LRTB,   BTC,   330,   435,   215,   226, STR_NETWORK_REFRESH,              STR_NETWORK_REFRESH_TIP},          // NGWW_REFRESH
 
+{ WWT_PUSHTXTBTN,   RESIZE_LRTB,   BTC,   215,   315,   197,   208, STR_NEWGRF_DOWNLOAD_BUTTON,       STR_NULL},                         // NGWW_NEWGRF_DL
 { WWT_PUSHTXTBTN,   RESIZE_LRTB,   BTC,   330,   435,   197,   208, STR_NEWGRF_SETTINGS_BUTTON,       STR_NULL},                         // NGWW_NEWGRF
 
 /* BOTTOM */
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/network/newgrf_download.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/network/newgrf_download.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -0,0 +1,784 @@
+/* $Id$ */
+
+/** @file newgrf_download.cpp GUI to download NewGRFs. */
+#include "../stdafx.h"
+#include "../openttd.h"
+#include "../strings_func.h"
+#include "../date_func.h"
+#include "../gui.h"
+#include "../window_gui.h"
+#include "../textbuf_gui.h"
+#include "../variables.h"
+#include "../newgrf.h"
+#include "newgrf_download.h"
+#include "core/config.h"
+#include "core/game.h"
+#include "../functions.h"
+#include "../debug.h"
+#include "../window_func.h"
+#include "../core/alloc_func.hpp"
+#include "../string_func.h"
+#include "../gfx_func.h"
+#include "../querystring_gui.h"
+#include "../sortlist_type.h"
+#include "../fileio.h"
+#include "../rev.h"
+
+// XXX: ...
+#include <curl/curl.h>
+
+#include "table/strings.h"
+#include "../table/sprites.h"
+
+static int _has_curl_init = 0;
+
+static CURLcode do_init_curl () {
+    CURLcode err;
+
+    if (!_has_curl_init) {
+        // XXX: be smarter about what to initialize
+        if ((err = curl_global_init(CURL_GLOBAL_ALL)))
+            return err;
+
+        _has_curl_init = 1;
+    }
+
+    return CURLE_OK;
+}
+
+struct check_write_ctx {
+    char *buf;
+    size_t len;
+    size_t offset;
+};
+
+#define QUERY_ITEM_SIZE (                   \
+            4   /* the "grf=" prefix */     \
+          + 8   /* the GRFID, in hex */     \
+          + 1   /* the ":" */               \
+          + 32  /* the md5sum, in hex */    \
+          + 1   /* the "&" or '\0' */       \
+        )
+
+static size_t check_available_write_cb (void *ptr, size_t size, size_t nmemb, void *arg) {
+    struct check_write_ctx *ctx = (struct check_write_ctx *) arg;
+    
+    // total number of bytes
+    size_t len = size * nmemb, oldlen = ctx->len;
+    
+    // increase the size until the new data fits
+    while (ctx->offset + len + 1 > ctx->len) {
+        ctx->len *= 2;
+    }
+    
+    // realloc if needed
+    if (ctx->len != oldlen)
+        ctx->buf = ReallocT<char>(ctx->buf, ctx->len);
+    
+    // thankfully we only have a limited amount of data..
+    memcpy(ctx->buf + ctx->offset, ptr, len);
+    ctx->offset += len;
+    *(ctx->buf + ctx->offset) = '\0';
+
+    return len;
+}
+
+/** Ask the master database about the availablity of the given NewGRFs, and update
+ * the GRFConfigs that are avilable to GCS_AVAILABLE
+ * @param config pointer to a linked-list of grfconfig's needed */
+static int CheckAvailableNewGRFs(GRFConfig *list)
+{
+    GRFConfig *c;
+    int i;
+    char md5buf[64];
+    struct check_write_ctx write_ctx;
+    CURL *curl;
+    char api_url[512], useragent[512];
+    char curl_errbuf[CURL_ERROR_SIZE];
+    long http_code;
+    char *buf_ptr;
+    CURLcode err;
+    int ret = -1;
+    int found = 0;
+
+    // initialize to a safe state for error handling
+    curl = write_ctx.buf = NULL;
+    
+    // count how many NewGRFs in our list
+    for (c = list, i = 0; c != NULL; c = c->next, i++) {}
+    
+    // how many bytes these take up...
+    int query_size = i * QUERY_ITEM_SIZE;
+    
+    // allocate the buffer for the query data
+    char *query_buf = MallocT<char>(query_size), *query_ptr = query_buf;
+    
+    // build the query
+    for (c = list; c != NULL; c = c->next) {
+        
+        // the "&", except not for the first entry
+        if (c != list)
+            *(query_ptr++) = '&';
+        
+        // format the md5sum
+        md5sumToString(md5buf, lastof(md5buf), c->md5sum);
+
+        // snprintf the entry into the query buffer
+        assert(snprintf(query_ptr, QUERY_ITEM_SIZE, "grf=%08X:%32s", BSWAP32(c->grfid), md5buf) == QUERY_ITEM_SIZE -1);
+        
+        // advance the query_ptr to point at the nul byte
+        query_ptr += QUERY_ITEM_SIZE - 1;
+    }
+
+    // buil the target URL by formatting NETWORK_NEWGRF_MASTER_API with the method as the only parameter
+    assert(snprintf(api_url, sizeof(api_url), NETWORK_NEWGRF_MASTER_API, "search") < 512);
+    
+    // our useragent
+    assert(snprintf(useragent, sizeof(useragent), NETWORK_HTTP_USER_AGENT, _openttd_revision) < 512);
+
+    // receiving is so damn hard...
+    write_ctx.buf = MallocT<char>(512);
+    write_ctx.len = 512;
+    write_ctx.offset = 0;
+
+    // set up curl globally if needed
+    if ((err = do_init_curl())) {
+        DEBUG(grfdl, 0, "curl_global_init failed: %s", curl_easy_strerror(err));
+        goto error;
+    }
+    
+    // allocate a new curl_easy
+    if ((curl = curl_easy_init()) == NULL) {
+        DEBUG(grfdl, 0, "curl_easy_init failed");
+        goto error;
+    }
+
+    // let's set some options...
+    if (
+            (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_VERBOSE, 1)))        // XXX: tie this in with OpenTTD's debug stuff
+    /*            
+            curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, XXX)
+    */            
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1)))             // no progress meter on stdout, thx
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)))  // XXX: human-readable error messages
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_URL, api_url)))              // the URL to retrieve
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_USERAGENT, useragent)))      // the User-agent to use
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, query_buf)))     // the POST data
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &check_available_write_cb)))
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &write_ctx)))     // the write buffer
+    ) {
+        DEBUG(grfdl, 0, "curl_easy_setopt failed: %s", curl_easy_strerror(err));
+        goto error;
+    }
+
+    // XXX: currently this freezes the UI..
+    if ((err = curl_easy_perform(curl))) {
+        DEBUG(grfdl, 0, "the curl request failed: %s", curl_easy_strerror(err));
+        goto error;
+    }
+    
+    // get the HTTP return code
+    if ((err = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code))) {
+        DEBUG(grfdl, 0, "curl_easy_getinfo failed: %s", curl_easy_strerror(err));
+        goto error;
+    }
+
+    if (http_code != 200) {
+        DEBUG(grfdl, 0, "Bad HTTP response code: %d", http_code);
+        goto error;
+    }
+
+    // start parsing...
+    buf_ptr = write_ctx.buf;
+
+    while (buf_ptr < write_ctx.buf + write_ctx.offset) {
+        // replace the \n with a \0
+        char *nl = strchr(buf_ptr, '\n');
+
+        if (!nl) {
+            DEBUG(grfdl, 0, "Incomplete line in search response: %s", buf_ptr);
+            goto error;
+        }
+
+        *nl = '\0';
+        
+        // check the format
+        if (buf_ptr[8] != ':' || buf_ptr[8 + 1 + 32] != ':') {
+            DEBUG(grfdl, 0, "Invalid line in search response: %s", buf_ptr);
+            goto error;
+        }
+
+        // separate out the fields
+        buf_ptr[8] = buf_ptr[8 + 1 + 32] = '\0';
+        
+        // parse the GRFID...
+        char *invalid;
+        uint32 grfid = strtol(buf_ptr + 0, &invalid, 16);
+
+        if (*invalid) {
+            DEBUG(grfdl, 0, "Invalid GRFID in search response: %s", buf_ptr);
+            goto error;
+        }
+        
+        // parse the md5sum
+        uint8 md5sum[16]; 
+
+        if (StringToMd5sum(md5sum, buf_ptr + 8 + 1)) {
+            DEBUG(grfdl, 0, "Invalid md5sum in search response: %s", buf_ptr);
+            goto error;
+        }
+
+        // copy the URL
+        char *url = strdup(buf_ptr + 8 + 1 + 32 + 1);
+
+        // look for the GRF
+        for (c = list; c != NULL; c = c->next) {
+            if (BSWAP32(c->grfid) == grfid && memcmp(c->md5sum, md5sum, 16) == 0) {
+                DEBUG(grfdl, 0, "Found %08X:%s at %s", buf_ptr + 0, buf_ptr + 8 + 1, url);
+
+                c->filename = url;
+                c->status = GCS_AVAILABLE;
+                ClrBit(c->status, GCF_COMPATIBLE);
+
+                found++;
+
+                break;
+            } 
+        }
+        
+        // free the allocated memory if it wasn't stored in a GRFConfig
+        if (c == NULL) {
+            free(url);
+        }
+
+        // move on to the next line
+        buf_ptr = nl + 1;
+    }
+    
+    // return number of NewGRFs found, defaults to -1 for errors
+    ret = found;    
+
+error :
+    // cleanup
+    free(write_ctx.buf);
+
+    if (curl)
+        curl_easy_cleanup(curl);
+    
+    // return as defined by ret
+    return ret;
+}
+
+/** Download the NewGRFs that are listed as GCS_AVAILABLE
+ * @param config pointer to a linked-list of grfconfig's to be downloaded */
+static int DownloadAvailableNewGRFs(GRFConfig *list)
+{
+    GRFConfig *c;
+    CURL *curl = NULL;
+    CURLcode err;
+    FILE *fh = NULL;
+    char md5buf[64], filename[256], useragent[512];
+    long http_code;
+    int ret = -1;
+    int found = 0;
+
+    // set up curl
+    if ((err = do_init_curl())) {
+        DEBUG(grfdl, 0, "curl_global_init failed: %s", curl_easy_strerror(err));
+        goto error;
+    }
+    
+    // allocate a new curl_easy
+    if ((curl = curl_easy_init()) == NULL) {
+        DEBUG(grfdl, 0, "curl_easy_init failed");
+        goto error;
+    }
+    
+    // our useragent
+    assert(snprintf(useragent, sizeof(useragent), NETWORK_HTTP_USER_AGENT, _openttd_revision) < 512);
+
+    // let's set some options...
+    if (
+            (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_VERBOSE, 1)))        // XXX: tie this in with OpenTTD's debug stuff
+    /*            
+            curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, XXX)
+    */            
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1)))             // no progress meter on stdout, thx
+//        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)))  // human-readable error messages
+
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_USERAGENT, useragent)))      // the User-agent to use
+
+/*       
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, query_buf)))     // the POST data
+        ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &check_available_write_cb)))
+*/        
+    ) {
+        DEBUG(grfdl, 0, "curl_easy_setopt failed: %s", curl_easy_strerror(err));
+        goto error;
+    }
+    
+    // then start looping through the NewGRFs to download
+    for (c = list; c != NULL; c = c->next) {
+        // skip the ones that are not available
+        if (c->status != GCS_AVAILABLE) 
+            continue;
+        
+        // format the md5sum
+        md5sumToString(md5buf, lastof(md5buf), c->md5sum);
+
+        // snprintf the entry into the query buffer
+        assert(snprintf(filename, sizeof(filename), "%08X-%32s.tar", BSWAP32(c->grfid), md5buf) < 256);
+
+        // open the file
+        if ((fh = FioFOpenFile(filename, "wb", CACHE_DIR)) == NULL) {
+            DEBUG(grfdl, 0, "Opening local GRF file failed: %s", filename);
+            goto error;
+        }
+        
+        // set up curl
+        if (
+                (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_URL, c->filename)))              // the URL to retrieve
+            ||  (CURLE_OK != (err = curl_easy_setopt(curl, CURLOPT_WRITEDATA, fh)))                 // XXX: need to fwrite() ourself
+        ) {
+            DEBUG(grfdl, 0, "curl_easy_setopt failed: %s", curl_easy_strerror(err));
+            goto error;
+        }
+        
+        // XXX: currently this freezes the UI..
+        if ((err = curl_easy_perform(curl))) {
+            DEBUG(grfdl, 0, "the curl request failed: %s", curl_easy_strerror(err));
+            goto error;
+        }
+        
+        // get the HTTP return code
+        if ((err = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code))) {
+            DEBUG(grfdl, 0, "curl_easy_getinfo failed: %s", curl_easy_strerror(err));
+            goto error;
+        }
+
+        if (http_code != 200) {
+            DEBUG(grfdl, 0, "Bad HTTP response code: %d", http_code);
+            goto error;
+        }
+
+        // OK, great
+        c->status = GCS_UNKNOWN;
+        ClrBit(c->status, GCF_COMPATIBLE);
+
+        found++;
+    }
+
+    // success
+    ret = found;
+
+error:
+    if (fh)
+        fclose(fh);
+    
+    if (curl)
+        curl_easy_cleanup(curl);
+
+    return ret;    
+}
+
+// XXX: copy-pasted from newgrf_gui.cpp
+static void ShowDownloadNewGRFInfo(const GRFConfig *c, uint x, uint y, uint w, uint bottom, bool show_params)
+{
+	char buff[256];
+
+	if (c->error != NULL) {
+		char message[512];
+		if (c->error->custom_message == NULL) {
+			SetDParamStr(0, c->filename);
+			SetDParamStr(1, c->error->data);
+			for (uint i = 0; i < c->error->num_params; i++) {
+				uint32 param = 0;
+				byte param_number = c->error->param_number[i];
+
+				if (param_number < c->num_params) param = c->param[param_number];
+
+				SetDParam(2 + i, param);
+			}
+
+			GetString(message, c->error->message, lastof(message));
+		} else {
+			SetDParamStr(0, c->error->custom_message);
+			GetString(message, STR_JUST_RAW_STRING, lastof(message));
+		}
+
+		SetDParamStr(0, message);
+		y += DrawStringMultiLine(x, y, c->error->severity, w, bottom - y);
+	}
+
+	/* Draw filename or not if it is not known (GRF sent over internet) */
+	if (c->filename != NULL) {
+		SetDParamStr(0, c->filename);
+		y += DrawStringMultiLine(x, y, STR_NEWGRF_FILENAME, w, bottom - y);
+	}
+
+	/* Prepare and draw GRF ID */
+	snprintf(buff, lengthof(buff), "%08X", BSWAP32(c->grfid));
+	SetDParamStr(0, buff);
+	y += DrawStringMultiLine(x, y, STR_NEWGRF_GRF_ID, w, bottom - y);
+
+	/* Prepare and draw MD5 sum */
+	md5sumToString(buff, lastof(buff), c->md5sum);
+	SetDParamStr(0, buff);
+	y += DrawStringMultiLine(x, y, STR_NEWGRF_MD5SUM, w, bottom - y);
+
+	/* Show GRF parameter list */
+	if (show_params) {
+		if (c->num_params > 0) {
+			GRFBuildParamList(buff, c, lastof(buff));
+			SetDParam(0, STR_JUST_RAW_STRING);
+			SetDParamStr(1, buff);
+		} else {
+			SetDParam(0, STR_01A9_NONE);
+		}
+		y += DrawStringMultiLine(x, y, STR_NEWGRF_PARAMETER, w, bottom - y);
+	}
+
+	/* Show flags */
+	if (c->status == GCS_NOT_FOUND)        y += DrawStringMultiLine(x, y, STR_NEWGRF_NOT_FOUND, w, bottom - y);
+	if (c->status == GCS_DISABLED)         y += DrawStringMultiLine(x, y, STR_NEWGRF_DISABLED, w, bottom - y);
+	if (HasBit(c->flags, GCF_COMPATIBLE)) y += DrawStringMultiLine(x, y, STR_NEWGRF_COMPATIBLE_LOADED, w, bottom - y);
+
+	/* Draw GRF info if it exists */
+	if (c->info != NULL && !StrEmpty(c->info)) {
+		SetDParam(0, STR_JUST_RAW_STRING);
+		SetDParamStr(1, c->info);
+		y += DrawStringMultiLine(x, y, STR_02BD, w, bottom - y);
+	} else {
+		y += DrawStringMultiLine(x, y, STR_NEWGRF_NO_INFO, w, bottom - y);
+	}
+}
+
+/**
+ * Window for downloading NewGRF files
+ */
+struct NewGRFDownloadWindow : public Window {
+	/* Names of the add a newgrf window widgets */
+	enum DownloadNewGRFWindowWidgets {
+        DNGRFS_CLOSEBOX = 0,
+		DNGRFS_CAPTION,
+		DNGRFS_BACKGROUND,
+		DNGRFS_FILE_LIST,
+		DNGRFS_SCROLLBAR,
+		DNGRFS_NEWGRF_INFO,
+		DNGRFS_CHECK_AVAILABLE,
+		DNGRFS_DOWNLOAD_AVAILABLE,
+		DNGRFS_RESIZE,
+	};
+    
+    struct NetworkGameInfo *info;   ///< the server we are downloading NewGRFs for
+	GRFConfig *list;        ///< temporary grf list to which changes are made
+	const GRFConfig *sel;
+
+	NewGRFDownloadWindow(const WindowDesc *desc, struct NetworkGameInfo *info, GRFConfig *list) : Window(desc, 0)
+	{
+        // resize magic
+		this->resize.step_height = 14;
+        
+        // the server info
+        this->info = info;
+
+        // the list of GRFs that we need
+        this->list = list;
+        
+        // display window?
+		this->FindWindowPlacementAndResize(desc);
+
+        // set up our own state
+		this->SetupDownloadNewGRFWindow();
+	}
+
+	~NewGRFDownloadWindow()
+	{
+        // Apply changes from list to the config
+        GRFConfig *config, *list;
+
+        // default to true, set back to false if we find a still-missing NewGRF
+        this->info->compatible = true;
+        
+        for (config = this->info->grfconfig, list = this->list; config != NULL && list != NULL; config = config->next, list = list->next) {
+            // skip those entries that aren't in list
+            if (config->grfid == list->grfid) {
+                // changed?
+                if (config->status != list->status && list->status != GCS_AVAILABLE) {
+                    config->status = list->status;
+                    ClrBit(config->flags, GCF_COMPATIBLE);
+                }
+            }
+            
+            // is this still NOT_FOUND?
+			if (config->status == GCS_NOT_FOUND) 
+                this->info->compatible = false;
+
+        }            
+
+		/* Remove the temporary copy of grf-list used in window */
+		ClearGRFConfigList(&this->list);
+    }
+
+    void SetupDownloadNewGRFWindow()
+	{
+		const GRFConfig *c;
+		int i;
+        
+        // count how many NewGRFs in our list
+		for (c = this->list, i = 0; c != NULL; c = c->next, i++) {}
+        
+        // something to do with vertical scrolling...
+		this->vscroll.cap = (this->widget[DNGRFS_FILE_LIST].bottom - this->widget[DNGRFS_FILE_LIST].top) / 14 + 1;
+		SetVScrollCount(this, i);
+        
+        // disable the "Check Available" button if there are no incompatible NewGRFs
+		this->SetWidgetDisabledState(DNGRFS_CHECK_AVAILABLE, (i == 0));
+
+        // disable the "Download Available" button until we've checked for available NewGRFs
+		this->SetWidgetDisabledState(DNGRFS_DOWNLOAD_AVAILABLE, 1);
+	}
+
+    virtual void OnPaint()
+	{
+        // superclass widget-drawing
+		this->DrawWidgets();
+
+		/* Draw the NewGRF file list */
+
+		int y = this->widget[DNGRFS_FILE_LIST].top;
+		int i = 0;
+
+        // for each GRF...
+		for (const GRFConfig *c = this->list; c != NULL; c = c->next, i++) {
+
+            // is scrolled into view
+			if (i >= this->vscroll.pos && i < this->vscroll.pos + this->vscroll.cap) {
+
+				SpriteID pal;
+				byte txtoffset;
+
+                // the NewGRF name
+				const char *text = (c->name != NULL && !StrEmpty(c->name)) ? c->name : c->filename;
+
+				/* 
+                 * Pick a colour 
+                 *
+                 *  orange  - server has it, we don't, haven't checked availablity yet
+                 *  red     - not available
+                 *  green   - available
+                 *  blue    - downloaded
+                 */
+				switch (c->status) {
+					case GCS_NOT_FOUND:
+                        pal = PALETTE_TO_ORANGE;
+                        break;
+
+					case GCS_DISABLED:
+						pal = PALETTE_TO_RED;
+						break;
+
+					case GCS_AVAILABLE:
+						pal = PALETTE_TO_GREEN;
+						break;
+
+					default:    // GCS_INITIALIZED or GCS_ACTIVATED
+						pal = PALETTE_TO_BLUE;
+						break;
+				}
+
+            /*
+                 XXX: flags?
+				if (pal != PALETTE_TO_RED) {
+					if (HasBit(c->flags, GCF_STATIC)) {
+						pal = PALETTE_TO_GREY;
+
+					} else if (HasBit(c->flags, GCF_COMPATIBLE)) {
+						pal = PALETTE_TO_ORANGE;
+
+					}
+				}
+            */
+
+                // draw the square in the chosen colour
+				DrawSprite(SPR_SQUARE, pal, 5, y + 2);
+
+                // XXX: hmm... use for download errors?
+				if (c->error != NULL) DrawSprite(SPR_WARNING_SIGN, 0, 20, y + 2);
+
+                // where to draw the text
+				txtoffset = c->error != NULL ? 35 : 25;
+
+                // draw the chosen text (GRF name/filename)
+				DoDrawStringTruncated(text, txtoffset, y + 3, this->sel == c ? TC_WHITE : TC_BLACK, this->width - txtoffset - 10);
+
+                // advance on to the next row
+				y += 14;
+			}
+		}
+
+		if (this->sel != NULL) {
+			/* Draw NewGRF file info */
+			const Widget *wi = &this->widget[DNGRFS_NEWGRF_INFO];
+    
+			ShowDownloadNewGRFInfo(this->sel, wi->left + 2, wi->top + 2, wi->right - wi->left - 2, wi->bottom, false);
+		}
+	}
+
+	virtual void OnClick(Point pt, int widget)
+	{
+        int found;
+
+		switch (widget) {
+            case DNGRFS_CHECK_AVAILABLE:
+                // do the HTTP request etc.
+                found = CheckAvailableNewGRFs(this->list);
+
+                if (found < 0) {
+                    // failed
+                    ShowErrorMessage(INVALID_STRING_ID, STR_NEWGRF_DOWNLOAD_CHECK_FAILED, 0, 0);
+
+                } else if (found == 0) {
+                    // no NewGRFs found
+                    ShowErrorMessage(INVALID_STRING_ID, STR_NEWGRF_DOWNLOAD_CHECK_NONE, 0, 0);
+
+                } else {
+                    // success, flip the button enabled states
+                    this->SetWidgetDisabledState(DNGRFS_CHECK_AVAILABLE, 1);
+                    this->SetWidgetDisabledState(DNGRFS_DOWNLOAD_AVAILABLE, 0);
+                    
+                    // redraw with new status
+                    this->SetDirty();
+                }
+				
+
+                break;
+
+            case DNGRFS_DOWNLOAD_AVAILABLE:
+                // do a bunch of HTTP requests
+                found = DownloadAvailableNewGRFs(this->list);
+
+                if (found < 0) {
+                    // failed
+                    ShowErrorMessage(INVALID_STRING_ID, STR_NEWGRF_DOWNLOAD_FAILED, 0, 0);
+
+                } else {
+                    if (found == 0) {
+                        // ???
+                        // this shouldn't really happen, as we check for this in CHECK_AVAILABLE
+                        ShowErrorMessage(INVALID_STRING_ID, STR_NEWGRF_DOWNLOAD_CHECK_NONE, 0, 0);
+
+                    } else {
+                        /* XXX: not exported, so prototype here */
+                        void ScanForTarFiles ();
+
+                        // load the newly downloaded NewGRF files
+                        ScanForTarFiles();
+                        ScanNewGRFFiles();
+                    } 
+
+                    // disable the button, not useful anymore
+                    this->SetWidgetDisabledState(DNGRFS_DOWNLOAD_AVAILABLE, 1);
+                    
+                    // redraw with new status
+                    this->SetDirty();
+                }   
+               
+                break;
+
+			case DNGRFS_FILE_LIST: { // Select a GRF
+				GRFConfig *c;
+				uint i = (pt.y - this->widget[DNGRFS_FILE_LIST].top) / 14 + this->vscroll.pos;
+
+				for (c = this->list; c != NULL && i > 0; c = c->next, i--) {}
+				this->sel = c;
+
+				this->SetDirty();
+				break;
+			}
+		}
+	}
+
+    virtual void OnResize(Point new_size, Point delta)
+	{
+		if (delta.x != 0) {
+			ResizeButtons(this, DNGRFS_CHECK_AVAILABLE, DNGRFS_DOWNLOAD_AVAILABLE);
+		}
+
+		this->vscroll.cap += delta.y / 14;
+		this->widget[DNGRFS_FILE_LIST].data = (this->vscroll.cap << 8) + 1;
+
+		this->SetupDownloadNewGRFWindow();
+	}
+
+	virtual void OnInvalidateData(int data = 0)
+	{
+		this->SetupDownloadNewGRFWindow();
+	}
+};
+
+/* Widget definition of the download newgrfs window */
+static const Widget _newgrf_download_widgets[] = {
+{   WWT_CLOSEBOX,  RESIZE_NONE, 10,   0,  10,   0,  13, STR_00C5,                    STR_018B_CLOSE_WINDOW },            // DNGRFS_CLOSEBOX
+{    WWT_CAPTION, RESIZE_RIGHT, 10,  11, 299,   0,  13, STR_NEWGRF_DOWNLOAD_CAPTION, STR_018C_WINDOW_TITLE_DRAG_THIS },  // DNGRFS_CAPTION
+{      WWT_PANEL, RESIZE_RIGHT, 10,   0, 299,  14,  29, STR_NULL,                    STR_NULL },                         // DNGRFS_BACKGROUND
+
+    /*
+{ WWT_PUSHTXTBTN,  RESIZE_NONE,  3,  10,  79,  16,  27, STR_NEWGRF_ADD,              STR_NEWGRF_ADD_TIP },               // SNGRFS_ADD
+{ WWT_PUSHTXTBTN,  RESIZE_NONE,  3,  80, 149,  16,  27, STR_NEWGRF_REMOVE,           STR_NEWGRF_REMOVE_TIP },            // SNGRFS_REMOVE
+{ WWT_PUSHTXTBTN,  RESIZE_NONE,  3, 150, 219,  16,  27, STR_NEWGRF_MOVEUP,           STR_NEWGRF_MOVEUP_TIP },            // SNGRFS_MOVE_UP
+{ WWT_PUSHTXTBTN, RESIZE_RIGHT,  3, 220, 289,  16,  27, STR_NEWGRF_MOVEDOWN,         STR_NEWGRF_MOVEDOWN_TIP },          // SNGRFS_MOVE_DOWN
+    */
+
+{     WWT_MATRIX,    RESIZE_RB, 10,   0, 287,  30,  99, 0x501,                       STR_NEWGRF_DOWNLOAD_LIST_TIP },     // DNGRFS_FILE_LIST
+{  WWT_SCROLLBAR,   RESIZE_LRB, 10, 288, 299,  30,  99, 0x0,                         STR_0190_SCROLL_BAR_SCROLLS_LIST }, // DNGRFS_SCROLLBAR
+{      WWT_PANEL,   RESIZE_RTB, 10,   0, 299, 100, 212, STR_NULL,                    STR_NULL },                         // DNGRFS_NEWGRF_INFO
+{ WWT_PUSHTXTBTN,    RESIZE_TB, 10,   0, 143, 213, 224, STR_NEWGRF_CHECK_AVAILABLE,  STR_NULL },                         // DNGRFS_CHECK_AVILABLE
+{ WWT_PUSHTXTBTN,   RESIZE_RTB, 10, 144, 287, 213, 224, STR_NEWGRF_DOWNLOAD_AVAILABLE,  STR_NULL },                      // DNGRFS_DOWNLOAD_AVAILABLE
+{  WWT_RESIZEBOX,  RESIZE_LRTB, 10, 288, 299, 213, 224, 0x0,                         STR_RESIZE_BUTTON },                // DNGRFS_RESIZE
+{ WIDGETS_END },
+};
+
+/* Window definition of the manage newgrfs window */
+static const WindowDesc _newgrf_download_desc = {
+	WDP_CENTER, WDP_CENTER, 300, 225, 300, 225,
+	WC_GAME_OPTIONS, WC_NONE,
+	WDF_STD_TOOLTIPS | WDF_STD_BTN | WDF_DEF_WIDGET | WDF_UNCLICK_BUTTONS | WDF_RESIZABLE,
+	_newgrf_download_widgets,
+};
+
+
+/** Setup the NewGRF download gui
+ * @param config pointer to a linked-list of grfconfig's needed */
+void ShowNewGRFDownload(struct NetworkGameInfo *info)
+{
+    // filter out those that we don't need to download
+    GRFConfig *head = NULL, **item = &head;
+    const GRFConfig *src;
+    
+    for (src = info->grfconfig; src != NULL; src = src->next) {
+        if (src->status == GCS_NOT_FOUND || HasBit(src->flags, GCF_COMPATIBLE)) {
+            // copy-past from newgrf_config.cpp CopyGRFConfigList
+            GRFConfig *c = CallocT<GRFConfig>(1);
+            *c = *src;
+            c->next = NULL;
+
+            if (src->filename  != NULL) c->filename  = strdup(src->filename);
+            if (src->name      != NULL) c->name      = strdup(src->name);
+            if (src->info      != NULL) c->info      = strdup(src->info);
+            if (src->error     != NULL) {
+                c->error = CallocT<GRFError>(1);
+                memcpy(c->error, src->error, sizeof(GRFError));
+                if (src->error->data != NULL) c->error->data = strdup(src->error->data);
+                if (src->error->custom_message != NULL) c->error->custom_message = strdup(src->error->custom_message);
+            }
+            
+            *item = c;
+            item = &c->next;
+        }
+    }
+
+	DeleteWindowByClass(WC_GAME_OPTIONS);
+	new NewGRFDownloadWindow(&_newgrf_download_desc, info, head);
+}
+
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/network/newgrf_download.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/network/newgrf_download.h	Thu Jul 24 16:18:29 2008 +0300
@@ -0,0 +1,21 @@
+/* $Id$ */
+
+/** @file newgrf_download.h Functions to download NewGRFs. */
+
+#ifndef NEWGRF_DOWNLOAD_H
+#define NEWGRF_DOWNLOAD_H
+
+#include "../newgrf_config.h"
+#include "core/game.h"
+
+// URL to the Master NewGRF database API
+#define NETWORK_NEWGRF_MASTER_API "http://skrblz.fixme.fi:8182/openttd-api/%s"
+
+// our HTTP User-Agent :)
+#define NETWORK_HTTP_USER_AGENT "OpenTTD (%s)"
+
+/* In newgrf_download.cpp */
+void ShowNewGRFDownload(struct NetworkGameInfo *info);
+
+#endif /* NEWGRF_DOWNLOAD_H */
+
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/newgrf_config.cpp
--- a/src/newgrf_config.cpp	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/newgrf_config.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -267,10 +267,14 @@
 	return res;
 }
 
-static bool ScanPathAddGrf(const char *filename)
+static bool ScanPathAddGrf(const char *filename, Subdirectory subdir)
 {
 	GRFConfig *c = CallocT<GRFConfig>(1);
 	c->filename = strdup(filename);
+    
+    // flag GRFs that are loaded from cache
+    if (subdir == CACHE_DIR)
+        SetBit(c->flags, GCF_CACHE);
 
 	bool added = true;
 	if (FillGRFDetails(c, false)) {
@@ -314,7 +318,7 @@
 }
 
 /* Scan a path for NewGRFs */
-static uint ScanPath(const char *path, size_t basepath_length)
+static uint ScanPath(const char *path, size_t basepath_length, Subdirectory subdir)
 {
 	extern bool FiosIsValidFile(const char *path, const struct dirent *ent, struct stat *sb);
 
@@ -337,7 +341,7 @@
 			/* Directory */
 			if (strcmp(d_name, ".") == 0 || strcmp(d_name, "..") == 0) continue;
 			AppendPathSeparator(filename, lengthof(filename));
-			num += ScanPath(filename, basepath_length);
+			num += ScanPath(filename, basepath_length, subdir);
 		} else if (sb.st_mode & S_IFREG) {
 			/* File */
 			char *ext = strrchr(filename, '.');
@@ -346,7 +350,7 @@
 			if (ext == NULL) continue;
 			if (strcasecmp(ext, ".grf") != 0) continue;
 
-			if (ScanPathAddGrf(filename + basepath_length)) num++;
+			if (ScanPathAddGrf(filename + basepath_length, subdir)) num++;
 		}
 	}
 
@@ -365,7 +369,7 @@
 	if (ext == NULL) return false;
 	if (strcasecmp(ext, ".grf") != 0) return false;
 
-	if (ScanPathAddGrf(filename)) num++;
+	if (ScanPathAddGrf(filename, (*tar).second.tar->subdir)) num++;
 
 	return num;
 }
@@ -398,7 +402,7 @@
 	DEBUG(grf, 1, "Scanning for NewGRFs");
 	FOR_ALL_SEARCHPATHS(sp) {
 		FioAppendDirectory(path, MAX_PATH, sp, DATA_DIR);
-		num += ScanPath(path, strlen(path));
+		num += ScanPath(path, strlen(path), DATA_DIR);
 	}
 	FOR_ALL_TARS(tar) {
 		num += ScanTar(tar);
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/newgrf_config.h
--- a/src/newgrf_config.h	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/newgrf_config.h	Thu Jul 24 16:18:29 2008 +0300
@@ -16,7 +16,7 @@
 	GCF_COPY,       ///< The data is copied from a grf in _all_grfs
 	GCF_INIT_ONLY,  ///< GRF file is processed up to GLS_INIT
 	GCF_RESERVED,   ///< GRF file passed GLS_RESERVE stage
-
+    GCF_CACHE,      ///< GRF file was loaded from cache
 };
 
 /** Status of GRF */
@@ -25,13 +25,14 @@
 	GCS_DISABLED,     ///< GRF file is disabled
 	GCS_NOT_FOUND,    ///< GRF file was not found in the local cache
 	GCS_INITIALISED,  ///< GRF file has been initialised
-	GCS_ACTIVATED     ///< GRF file has been activated
+	GCS_ACTIVATED,    ///< GRF file has been activated
+    GCS_AVAILABLE     ///< GRF was not found in the local cache, but the download repo has it
 };
 
 /** Status of post-gameload GRF compatibility check */
 enum GRFListCompatibility{
 	GLC_ALL_GOOD,   ///< All GRF needed by game are present
-	GLC_COMPATIBLE, ///< Compatible (eg. the same ID, but different chacksum) GRF found in at least one case
+	GLC_COMPATIBLE, ///< Compatible (eg. the same ID, but different checksum) GRF found in at least one case
 	GLC_NOT_FOUND   ///< At least one GRF couldn't be found (higher priority than GLC_COMPATIBLE)
 };
 
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/newgrf_gui.cpp
--- a/src/newgrf_gui.cpp	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/newgrf_gui.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -152,7 +152,9 @@
 		int n = 0;
 
 		/* Count the number of GRFs */
-		for (c = _all_grfs; c != NULL; c = c->next) n++;
+		for (c = _all_grfs; c != NULL; c = c->next) 
+            if (!HasBit(c->flags, GCF_CACHE))
+                n++;
 
 		this->vscroll.cap = (wl->bottom - wl->top) / 10;
 		SetVScrollCount(this, n);
@@ -164,7 +166,7 @@
 
 		uint y = wl->top + 1;
 		for (c = _all_grfs, n = 0; c != NULL && n < (this->vscroll.pos + this->vscroll.cap); c = c->next, n++) {
-			if (n >= this->vscroll.pos) {
+			if (!HasBit(c->flags, GCF_CACHE) && n >= this->vscroll.pos) {
 				bool h = c == this->sel;
 				const char *text = (c->name != NULL && !StrEmpty(c->name)) ? c->name : c->filename;
 
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/newgrf_house.cpp
--- a/src/newgrf_house.cpp	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/newgrf_house.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -69,12 +69,8 @@
 		IncreaseBuildingCount(town, house_id);
 		if (IsHouseCompleted(t)) town->population += GetHouseSpecs(house_id)->population;
 
-		/* Increase the number of houses for every house tile which
-		 * has a size bit set. Multi tile buildings have got only
-		 * one tile with such a bit set, so there is no problem. */
-		if (GetHouseSpecs(GetHouseType(t))->building_flags & BUILDING_HAS_1_TILE) {
-			town->num_houses++;
-		}
+		/* Increase the number of houses for every house, but only once. */
+		if (GetHouseNorthPart(house_id) == 0) town->num_houses++;
 	}
 
 	/* Update the population and num_house dependant values */
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/string.cpp
--- a/src/string.cpp	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/string.cpp	Thu Jul 24 16:18:29 2008 +0300
@@ -203,6 +203,28 @@
 	return p;
 }
 
+/** Convert the hexadecimal string representation to a binary md5sum
+ * @param md5sum where to put the md5sum into
+ * @param buf buffer containing a string of 32 hex chars
+ * @return 0 on success, -1 on failure */
+int StringToMd5sum(uint8 md5sum[16], const char *buf)
+{
+    char hex[3] = { 0, 0, 0};
+    char *invalid;
+
+	for (uint i = 0; i < 16; i++) {
+        hex[0] = buf[i * 2 + 0];
+        hex[1] = buf[i * 2 + 1];
+
+        md5sum[i] = strtol(hex, &invalid, 16);
+
+        if (*invalid)
+            return -1;
+	}
+
+	return 0;
+}
+
 
 /* UTF-8 handling routines */
 
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/string_func.h
--- a/src/string_func.h	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/string_func.h	Thu Jul 24 16:18:29 2008 +0300
@@ -54,6 +54,8 @@
 
 /** Convert the md5sum number to a 'hexadecimal' string, return next pos in buffer */
 char *md5sumToString(char *buf, const char *last, const uint8 md5sum[16]);
+/** Convert the hexademical string to a binary md5sum, return 0 on success, nonzero on error */
+int StringToMd5sum(uint8 md5sum[16], const char *buf);
 
 /**
  * Only allow certain keys. You can define the filter to be used. This makes
diff -r 3646f7f4fcfc -r 5036deb1fcaf src/tar_type.h
--- a/src/tar_type.h	Fri Jul 18 12:20:30 2008 +0000
+++ b/src/tar_type.h	Thu Jul 24 16:18:29 2008 +0300
@@ -11,6 +11,7 @@
 /** The define of a TarList. */
 struct TarListEntry {
 	const char *filename;
+    Subdirectory subdir;
 };
 
 struct TarFileListEntry {
