More tests!

src/htsname.c

@@ -41,6 +41,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsmd5.h"
 #include "htstools.h"
 #include "htscharset.h"
+#include "htsencoding.h"
 #include <ctype.h>
 
 #define ADD_STANDARD_PATH \
@@ -290,11 +291,14 @@ int url_savename(char *adr_complete, char *fil_complete, char *save,
     }
     fil = newfil;
   }
-  // Decode remaining %
-  strcpybuff(fil, unescape_http(catbuff, fil));
-  // , BUT do not decode high chars
-  //strcpybuff(fil,unescape_http_unharm(fil, 1));
-  // YES (not server side, but fs/client side)
+
+  // decode remaining % (normally not necessary; already done in htsparse.c)
+  if (hts_unescapeUrl(fil, catbuff, sizeof(catbuff)) == 0) {
+    strcpybuff(fil, catbuff);
+  } else {
+    hts_log_print(opt, LOG_WARNING,
+      "could not URL-decode string '%s'", fil);
+  }
 
 #if HTS_USEMMS
   /* .asx hack */

tests/11_crawl-idna.test

@@ -2,7 +2,12 @@
 #
 
 # unicode tests
-bash crawl-test.sh --errors 1 --files 5 \
+bash crawl-test.sh \
+	--errors 1 --files 5 \
+	--found 'café.ut.httrack.com/unicode-links/café3860.html' \
+	--found 'café.ut.httrack.com/unicode-links/café30f4.html' \
+	--found 'café.ut.httrack.com/unicode-links/café5e1f.html' \
+	--found 'café.ut.httrack.com/unicode-links/café7b30.html' \
 	httrack 'http://ut.httrack.com/unicode-links/idna.html' \
 	'+*.ut.httrack.com/*' --robots=0
 

tests/11_crawl-international.test

@@ -2,6 +2,48 @@
 #
 
 # unicode tests
-bash crawl-test.sh --errors 1 --files 10 httrack http://ut.httrack.com/unicode-links/utf8.html
-bash crawl-test.sh --errors 4 --files  7 httrack http://ut.httrack.com/unicode-links/default.html
-bash crawl-test.sh --errors 2 --files  9 httrack http://ut.httrack.com/unicode-links/iso88591.html
+bash crawl-test.sh \
+	--errors 1 --files 10 \
+	--found ut.httrack.com/unicode-links/caf%a91bce.html \
+	--found ut.httrack.com/unicode-links/café30f4.html \
+	--found ut.httrack.com/unicode-links/café3860.html \
+	--found ut.httrack.com/unicode-links/café463e.html \
+	--found ut.httrack.com/unicode-links/café5e1f.html \
+	--found ut.httrack.com/unicode-links/café7b30.html \
+	--found ut.httrack.com/unicode-links/café8007.html \
+	--found ut.httrack.com/unicode-links/café9fa8.html \
+	--found ut.httrack.com/unicode-links/caféae52.html \
+	--found ut.httrack.com/unicode-links/caféc009.html \
+	--found ut.httrack.com/unicode-links/utf8.html \
+	httrack http://ut.httrack.com/unicode-links/utf8.html
+
+bash crawl-test.sh \
+	--errors 4 --files  7 \
+	--found ut.httrack.com/unicode-links/café3860.html \
+	--found ut.httrack.com/unicode-links/café9fa8.html \
+	--found ut.httrack.com/unicode-links/café30f4.html \
+	--found ut.httrack.com/unicode-links/café5e1f.html \
+	--found ut.httrack.com/unicode-links/café7b30.html \
+	--found ut.httrack.com/unicode-links/café8007.html \
+	--found ut.httrack.com/unicode-links/caf%e939bd.html \
+	--found ut.httrack.com/unicode-links/caf%e9ae52.html \
+	--found ut.httrack.com/unicode-links/caféaec2.html \
+	--found ut.httrack.com/unicode-links/caféfad6.html \
+	--found ut.httrack.com/unicode-links/default.html \
+	httrack http://ut.httrack.com/unicode-links/default.html
+
+bash crawl-test.sh \
+	--errors 2 --files  9 \
+	--found ut.httrack.com/unicode-links/caf%a9ae52.html \
+	--found ut.httrack.com/unicode-links/caf%a9bf59.html \
+	--found ut.httrack.com/unicode-links/café30f4.html \
+	--found ut.httrack.com/unicode-links/café3860.html \
+	--found ut.httrack.com/unicode-links/café5e1f.html \
+	--found ut.httrack.com/unicode-links/café647f.html \
+	--found ut.httrack.com/unicode-links/café7b30.html \
+	--found ut.httrack.com/unicode-links/café8007.html \
+	--found ut.httrack.com/unicode-links/caféaec2.html \
+	--found ut.httrack.com/unicode-links/caféfad6.html \
+	--found ut.httrack.com/unicode-links/iso88591.html \
+	httrack http://ut.httrack.com/unicode-links/iso88591.html
+

tests/11_crawl-parsing.test

@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+
+# http://code.google.com/p/httrack/issues/detail?id=4&can=1
+bash crawl-test.sh --errors 0 --files 4 \
+	httrack http://ut.httrack.com/parsing/events.html
+
+# http://code.google.com/p/httrack/issues/detail?id=2&can=1
+bash crawl-test.sh --errors 0 --files 3 \
+	httrack http://ut.httrack.com/parsing/background-image.html
+

tests/crawl-test.sh

@@ -68,7 +68,7 @@ function start-crawl {
       ;;
     --no-purge|--summary)
       ;;
-    --errors|--files)
+    --errors|--files|--found|--not-found|--directory)
       pos=$[${pos}+1]
       test "$#" -ge "$pos" || warning "missing argument" || return 1
       ;;
@@ -127,12 +127,40 @@ function start-crawl {
       ;;
     --errors)
       shift
-      test "$#" -gt 0 || warning "missing argument" || return 1
       assert_equals "checking errors" "$1" "$(grep -iEc "^[0-9\:]*[[:space:]]Error:" "${tmp}/hts-log.txt")"
       ;;
+    --found)
+      shift
+      info "checking for $1"
+      if test -f "${tmp}/$1" ; then
+        result "OK"
+      else
+        result "not found"
+        exit 1
+      fi
+      ;;
+    --not-found)
+      shift
+      info "checking for $1"
+      if test -f "${tmp}/$1" ; then
+        result "OK"
+      else
+        result "not found"
+        exit 1
+      fi
+      ;;
+    --directory)
+      shift
+      info "checking for $1"
+      if test -d "${tmp}/$1" ; then
+        result "OK"
+      else
+        result "not found"
+        exit 1
+      fi
+      ;;
     --files)
       shift
-      test "$#" -gt 0 || warning "missing argument" || return 1
       nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${tmp}/hts-log.txt" \
         | sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
       assert_equals "checking files" "$1" "$nFiles"

tests/run-all-tests.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+
+error=0
+for i in *.test ; do
+	if bash $i ; then
+		echo "$i: passed" >&2
+	else
+		echo "$i: ERROR" >&2
+		error=1
+	fi
+done
+
+if test "$error" -eq 0; then
+	echo "all tests passed" >&2
+else
+	echo "one or more tests failed" >&2
+fi
+
+exit $error