mirror of
https://github.com/xroche/httrack.git
synced 2026-07-12 20:07:43 +03:00
Compare commits
3 Commits
drop-swf-j
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
627ae2b043 | ||
|
|
71ab3574ef | ||
|
|
076127ddea |
1
README
1
README
@@ -40,7 +40,6 @@ These are the principals limits of HTTrack for that moment. Note that we did not
|
||||
that would have solved them.
|
||||
|
||||
- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
|
||||
- Some java classes may not find some files on them (class included)
|
||||
- Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*
|
||||
|
||||
|
||||
|
||||
14
configure.ac
14
configure.ac
@@ -64,16 +64,6 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
|
||||
# Export version info
|
||||
AC_SUBST(VERSION_INFO)
|
||||
|
||||
# Versioned plugin name for dlopen() in hts_create_opt(); soname major is
|
||||
# libtool's current - age, so this tracks VERSION_INFO bumps automatically.
|
||||
HTS_SONAME_MAJOR=$((${VERSION_INFO%%:*} - ${VERSION_INFO##*:}))
|
||||
case "$host_os" in
|
||||
darwin*) HTS_LIBHTSJAVA_NAME="libhtsjava.$HTS_SONAME_MAJOR.dylib" ;;
|
||||
*) HTS_LIBHTSJAVA_NAME="libhtsjava.so.$HTS_SONAME_MAJOR" ;;
|
||||
esac
|
||||
AC_DEFINE_UNQUOTED([HTS_LIBHTSJAVA_NAME], ["$HTS_LIBHTSJAVA_NAME"],
|
||||
[Versioned libhtsjava runtime name, derived from VERSION_INFO])
|
||||
|
||||
### Default CFLAGS
|
||||
DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
|
||||
-Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
|
||||
@@ -119,8 +109,8 @@ case "$CFLAGS" in
|
||||
esac
|
||||
|
||||
# Force libc back into DT_NEEDED for libraries that reach it only through
|
||||
# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
|
||||
# linker; Apple ld rejects these flags and links libSystem unconditionally.
|
||||
# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
|
||||
# ld rejects these flags and links libSystem unconditionally.
|
||||
AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
|
||||
[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
|
||||
AC_SUBST([LIBC_FORCE_LINK])
|
||||
|
||||
5
debian/copyright
vendored
5
debian/copyright
vendored
@@ -7,9 +7,8 @@ Files: *
|
||||
Copyright: 1998-2026 Xavier Roche and other contributors
|
||||
License: GPL-3+
|
||||
Comment:
|
||||
The engine includes contributions from Yann Philippot (src/htsjava.c,
|
||||
src/htsjava.h). htsbasenet.h links against the system OpenSSL library
|
||||
(originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
|
||||
htsbasenet.h links against the system OpenSSL library (originally by Eric
|
||||
Young); no OpenSSL/SSLeay code is bundled here.
|
||||
|
||||
Files: src/minizip/*
|
||||
Copyright: 1998-2010 Gilles Vollant
|
||||
|
||||
1
debian/libhttrack-dev.files
vendored
1
debian/libhttrack-dev.files
vendored
@@ -1,5 +1,4 @@
|
||||
usr/include/httrack
|
||||
usr/lib/*/libhttrack.{so}
|
||||
usr/lib/*/libhtsjava.{so}
|
||||
usr/share/httrack/libtest
|
||||
usr/lib/*/httrack/libtest
|
||||
|
||||
1
debian/libhttrack3.files
vendored
1
debian/libhttrack3.files
vendored
@@ -1,3 +1,2 @@
|
||||
usr/lib/*/libhttrack.so.3*
|
||||
usr/lib/*/libhtsjava.so.3*
|
||||
usr/share/httrack/templates
|
||||
|
||||
@@ -609,10 +609,9 @@ most mirrors do work. We still are working to improve the mirror quality of HTTr
|
||||
<br>
|
||||
|
||||
</a><a NAME="Q2b">Q: <strong>Some Java applets do not work properly!</strong><br>
|
||||
A: <em>Java applets may not work in some cases, for example if HTTrack failed to detect all included classes
|
||||
or files called within the class file. Sometimes, Java applets need to be online, because remote files are
|
||||
directly caught. Finally, the site structure can be incompatible with the class (always try to keep the original site structure
|
||||
when you want to get Java classes)<br>
|
||||
A: <em>Java applets may not work in some cases, because HTTrack does not parse compiled class files to find
|
||||
the resources they load. Java applets often need to be online anyway, since remote files are fetched directly
|
||||
at runtime.<br>
|
||||
If there is no way to make some classes work properly, you can exclude them with the filters.
|
||||
They will be available, but only online.
|
||||
</em>
|
||||
|
||||
@@ -243,7 +243,7 @@ Build options:
|
||||
Spider options:
|
||||
bN accept cookies in cookies.txt (0=do not accept,* 1=accept) (--cookies[=N])
|
||||
u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (--check-type[=N])
|
||||
j *parse Java Classes (j0 don't parse) (--parse-java[=N])
|
||||
j *parse scripts (j0 don't parse) (--parse-java[=N])
|
||||
sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N])
|
||||
%h force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (--http-10)
|
||||
%B tolerant requests (accept bogus responses on some servers, but not standard!) (--tolerant)
|
||||
@@ -417,7 +417,7 @@ site. Specifically, the defauls are:
|
||||
X *purge old files after update (X0 keep delete)
|
||||
bN accept cookies in cookies.txt (0=do not accept,* 1=accept)
|
||||
u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
|
||||
j *parse Java Classes (j0 don't parse)
|
||||
j *parse scripts (j0 don't parse)
|
||||
sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
|
||||
C create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)
|
||||
f *log file mode
|
||||
@@ -497,9 +497,9 @@ about your browser retaining them.
|
||||
<p align=justify> This causes different document types to be analyzed
|
||||
differently.
|
||||
|
||||
<pre><b><i> j *parse Java Classes (j0 don't parse) </i></b></pre>
|
||||
<pre><b><i> j *parse scripts (j0 don't parse) </i></b></pre>
|
||||
|
||||
<p align=justify> This causes Java class files to be parsed looking for
|
||||
<p align=justify> This causes scripts to be parsed looking for
|
||||
URLs.
|
||||
|
||||
<pre><b><i> sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) </i></b></pre>
|
||||
@@ -1178,7 +1178,7 @@ accepted and sent back in order to allow access.
|
||||
Spider options:
|
||||
bN accept cookies in cookies.txt (0=do not accept,* 1=accept)
|
||||
u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
|
||||
j *parse Java Classes (j0 don't parse)
|
||||
j *parse scripts (j0 don't parse)
|
||||
sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
|
||||
%h force HTTP/1.0 requests (reduce update features, only for old servers or proxies)
|
||||
%B tolerant requests (accept bogus responses on some servers, but not standard!)
|
||||
@@ -2451,12 +2451,10 @@ work. We still are working to improve the mirror quality of HTTrack.
|
||||
|
||||
<p align=justify><b>Q: Some Java applets do not work properly! </b>
|
||||
|
||||
<p align=justify> A: Java applets may not work in some cases, for
|
||||
example if HTTrack failed to detect all included classes or files called
|
||||
within the class file. Sometimes, Java applets need to be online,
|
||||
because remote files are directly caught. Finally, the site structure
|
||||
can be incompatible with the class (always try to keep the original site
|
||||
structure when you want to get Java classes)
|
||||
<p align=justify> A: Java applets may not work in some cases, because
|
||||
HTTrack does not parse compiled class files to find the resources they
|
||||
load. Java applets often need to be online anyway, since remote files
|
||||
are fetched directly at runtime.
|
||||
|
||||
<p align=justify> If there is no way to make some classes work properly,
|
||||
you can exclude them with the filters. They will be available, but only
|
||||
|
||||
@@ -764,9 +764,9 @@ check, * u1 check but /, u2 check always)
|
||||
<td width="78%">
|
||||
|
||||
|
||||
<p>*parse Java Classes (j0 don t parse, bitmask: |1 parse
|
||||
default, |2 don t parse .class |4 don t parse .js |8 don t
|
||||
be aggressive) (--parse-java[=N])</p></td></tr>
|
||||
<p>*parse scripts (j0 don t parse, bitmask: |1 parse
|
||||
default, |4 don t parse .js |8 don t be aggressive)
|
||||
(--parse-java[=N])</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="11%"></td>
|
||||
<td width="6%">
|
||||
@@ -889,8 +889,8 @@ only) (--protocol[=N])</p></td></tr>
|
||||
<td width="78%">
|
||||
|
||||
|
||||
<p>disable a specific external mime module (-%w htsswf -%w
|
||||
htsjava) (--disable-module <param>)</p></td></tr>
|
||||
<p>disable a specific external mime module (-%w
|
||||
httrack-plugin) (--disable-module <param>)</p></td></tr>
|
||||
</table>
|
||||
|
||||
<p style="margin-left:11%; margin-top: 1em"><b>Browser
|
||||
@@ -2175,10 +2175,6 @@ Several scripts generating complex filenames may not find
|
||||
them (ex:
|
||||
img.src=’image’+a+Mobj.dst+’.gif’)</small></p>
|
||||
|
||||
<p style="margin-left:11%; margin-top: 1em"><small>- Some
|
||||
java classes may not find some files on them (class
|
||||
included)</small></p>
|
||||
|
||||
<p style="margin-left:11%; margin-top: 1em"><small>-
|
||||
Cgi-bin links may not work properly in some cases
|
||||
(parameters needed). To avoid them: use filters like
|
||||
|
||||
@@ -288,8 +288,8 @@ This option define whether the engine has to abandon a host if a timeout/"too sl
|
||||
&P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
|
||||
This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
|
||||
|
||||
j *parse Java Classes (j0 don't parse)
|
||||
This option define whether the engine has to parse java files or not to catch included files
|
||||
j *parse scripts (j0 don't parse)
|
||||
This option defines whether the engine has to parse scripts or not to catch included files
|
||||
|
||||
I *make an index (I0 don't make)
|
||||
This option define whether the engine has to generate an index.html on the top directory
|
||||
|
||||
413
html/plug.html
413
html/plug.html
@@ -103,220 +103,219 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<br>
|
||||
|
||||
You can write external functions to be plugged in the httrack library very easily.
|
||||
We'll see there some examples.
|
||||
You can write external functions to be plugged in the httrack library very easily.
|
||||
We'll see there some examples.
|
||||
|
||||
<br><br>
|
||||
|
||||
The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
|
||||
The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
|
||||
<ul>
|
||||
<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
|
||||
<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
|
||||
<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
|
||||
</ul>
|
||||
|
||||
<br>
|
||||
References:
|
||||
<ul>
|
||||
<li>the <tt>httrack-library.h</tt> prototype file
|
||||
<br />
|
||||
|
||||
The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
|
||||
The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
|
||||
<ul>
|
||||
<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
|
||||
<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
|
||||
<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
|
||||
</ul>
|
||||
|
||||
<br>
|
||||
References:
|
||||
<ul>
|
||||
<li>the <tt>httrack-library.h</tt> prototype file
|
||||
<br />
|
||||
Note: the <i>Initialization</i>, <i>Main functions</i>, <i>Options handling</i> and <i>Wrapper functions</i> sections are generally the only ones to be considered.
|
||||
</li>
|
||||
<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
|
||||
<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
|
||||
<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
|
||||
<li>the <tt>htsjava.c</tt> source file (the java class plugin ; overrides 'detect' and 'parse')</li>
|
||||
<li>the example given at the end of this document</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
Below the list of functions to be defined in the module (plugin).<br />
|
||||
<br />
|
||||
|
||||
<table width="100%">
|
||||
<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
|
||||
The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
|
||||
|
||||
<br />
|
||||
Wrappers can be plugged inside hts_plug() using:<br />
|
||||
<tt>
|
||||
CHAIN_FUNCTION(opt, <callback name>, <our callback function name>, <our callback function optional custom pointer argument>);
|
||||
</tt>
|
||||
<br />
|
||||
|
||||
<br />
|
||||
Example:
|
||||
<br />
|
||||
<tt>
|
||||
CHAIN_FUNCTION(opt, check_html, process, userdef);
|
||||
</tt>
|
||||
<br />
|
||||
|
||||
</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
|
||||
|
||||
<!-- -->
|
||||
|
||||
<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
|
||||
The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
|
||||
|
||||
</table>
|
||||
|
||||
|
||||
<br />
|
||||
Note that all callbacks (except init and uninit) take as first two argument:
|
||||
<ul>
|
||||
<li>the t_hts_callbackarg structure<br />
|
||||
this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
|
||||
</li>
|
||||
<li>the httrackp structure<br />
|
||||
this structure, holding all current httrack options and mirror state, can be read or mofidied
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
Below the list of callbacks, and associated external wrappers.
|
||||
|
||||
<table width="100%">
|
||||
<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
|
||||
|
||||
<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
|
||||
|
||||
<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
|
||||
|
||||
<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
|
||||
|
||||
|
||||
</table>
|
||||
<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
|
||||
<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
|
||||
<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
|
||||
<li>the example given at the end of this document</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
Below the list of functions to be defined in the module (plugin).<br />
|
||||
<br />
|
||||
|
||||
<table width="100%">
|
||||
<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
|
||||
The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
|
||||
|
||||
<br />
|
||||
Wrappers can be plugged inside hts_plug() using:<br />
|
||||
<tt>
|
||||
CHAIN_FUNCTION(opt, <callback name>, <our callback function name>, <our callback function optional custom pointer argument>);
|
||||
</tt>
|
||||
<br />
|
||||
|
||||
<br />
|
||||
Example:
|
||||
<br />
|
||||
<tt>
|
||||
CHAIN_FUNCTION(opt, check_html, process, userdef);
|
||||
</tt>
|
||||
<br />
|
||||
|
||||
</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
|
||||
|
||||
<!-- -->
|
||||
|
||||
<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
|
||||
The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
|
||||
|
||||
</table>
|
||||
|
||||
|
||||
<br />
|
||||
Note that all callbacks (except init and uninit) take as first two argument:
|
||||
<ul>
|
||||
<li>the t_hts_callbackarg structure<br />
|
||||
this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
|
||||
</li>
|
||||
<li>the httrackp structure<br />
|
||||
this structure, holding all current httrack options and mirror state, can be read or mofidied
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<br />
|
||||
Below the list of callbacks, and associated external wrappers.
|
||||
|
||||
<table width="100%">
|
||||
<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
|
||||
|
||||
<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
|
||||
|
||||
<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
|
||||
|
||||
<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
|
||||
<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
|
||||
|
||||
|
||||
</table>
|
||||
|
||||
<br><br>
|
||||
Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
|
||||
|
||||
Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
|
||||
|
||||
|
||||
<br />
|
||||
An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
|
||||
<table width="100%" border="2">
|
||||
<tr><td>
|
||||
<pre>
|
||||
/* system includes */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* standard httrack module includes */
|
||||
#include "httrack-library.h"
|
||||
#include "htsopt.h"
|
||||
#include "htsdefines.h"
|
||||
|
||||
/* local function called as "check_html" callback */
|
||||
static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt,
|
||||
/*other parameters are callback-specific*/
|
||||
char* html, int len, const char* url_address, const char* url_file) {
|
||||
void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/
|
||||
|
||||
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
|
||||
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
|
||||
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
|
||||
html, len, url_address, url_file)) {
|
||||
return 0; /* abort */
|
||||
}
|
||||
}
|
||||
|
||||
printf("file %s%s content: %s\n", url_address, url_file, html);
|
||||
return 1; /* success */
|
||||
}
|
||||
|
||||
/* local function called as "end" callback */
|
||||
static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
|
||||
void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/
|
||||
|
||||
/* processing */
|
||||
fprintf(stderr, "That's all, folks!\n");
|
||||
|
||||
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
|
||||
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
|
||||
/* status is ok on our side, return other callabck's status */
|
||||
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
|
||||
}
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
|
||||
/*
|
||||
module entry point
|
||||
the function name and prototype MUST match this prototype
|
||||
*/
|
||||
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
|
||||
/* optional argument passed in the commandline we won't be using here */
|
||||
const char *arg = strchr(argv, ',');
|
||||
if (arg != NULL)
|
||||
arg++;
|
||||
|
||||
/* plug callback functions */
|
||||
CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
|
||||
CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
|
||||
/*
|
||||
module exit point
|
||||
the function name and prototype MUST match this prototype
|
||||
*/
|
||||
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
|
||||
fprintf(stder, "Module unplugged");
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
</pre>
|
||||
</td></tr></table>
|
||||
|
||||
<br />
|
||||
Compile this file ; for example:
|
||||
<br />
|
||||
<tt>
|
||||
gcc -O -g3 -shared -o mylibrary.so myexample.c
|
||||
</tt>
|
||||
<br />
|
||||
and plug the module using the commandline ; for example:
|
||||
<br />
|
||||
<tt>
|
||||
httrack --wrapper mylibrary http://www.example.com
|
||||
</tt>
|
||||
<br />
|
||||
or, if some parameters are desired:
|
||||
<br />
|
||||
<tt>
|
||||
httrack --wrapper mylibrary,myparameter-string http://www.example.com
|
||||
</tt>
|
||||
<br />
|
||||
(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
|
||||
<br />
|
||||
|
||||
|
||||
<br />
|
||||
An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
|
||||
<table width="100%" border="2">
|
||||
<tr><td>
|
||||
<pre>
|
||||
/* system includes */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* standard httrack module includes */
|
||||
#include "httrack-library.h"
|
||||
#include "htsopt.h"
|
||||
#include "htsdefines.h"
|
||||
|
||||
/* local function called as "check_html" callback */
|
||||
static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt,
|
||||
/*other parameters are callback-specific*/
|
||||
char* html, int len, const char* url_address, const char* url_file) {
|
||||
void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/
|
||||
|
||||
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
|
||||
if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
|
||||
if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
|
||||
html, len, url_address, url_file)) {
|
||||
return 0; /* abort */
|
||||
}
|
||||
}
|
||||
|
||||
printf("file %s%s content: %s\n", url_address, url_file, html);
|
||||
return 1; /* success */
|
||||
}
|
||||
|
||||
/* local function called as "end" callback */
|
||||
static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
|
||||
void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg); /*optional user-defined arg*/
|
||||
|
||||
/* processing */
|
||||
fprintf(stderr, "That's all, folks!\n");
|
||||
|
||||
/* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
|
||||
if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
|
||||
/* status is ok on our side, return other callabck's status */
|
||||
return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
|
||||
}
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
|
||||
/*
|
||||
module entry point
|
||||
the function name and prototype MUST match this prototype
|
||||
*/
|
||||
EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
|
||||
/* optional argument passed in the commandline we won't be using here */
|
||||
const char *arg = strchr(argv, ',');
|
||||
if (arg != NULL)
|
||||
arg++;
|
||||
|
||||
/* plug callback functions */
|
||||
CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
|
||||
CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
|
||||
/*
|
||||
module exit point
|
||||
the function name and prototype MUST match this prototype
|
||||
*/
|
||||
EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
|
||||
fprintf(stder, "Module unplugged");
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
</pre>
|
||||
</td></tr></table>
|
||||
|
||||
<br />
|
||||
Compile this file ; for example:
|
||||
<br />
|
||||
<tt>
|
||||
gcc -O -g3 -shared -o mylibrary.so myexample.c
|
||||
</tt>
|
||||
<br />
|
||||
and plug the module using the commandline ; for example:
|
||||
<br />
|
||||
<tt>
|
||||
httrack --wrapper mylibrary http://www.example.com
|
||||
</tt>
|
||||
<br />
|
||||
or, if some parameters are desired:
|
||||
<br />
|
||||
<tt>
|
||||
httrack --wrapper mylibrary,myparameter-string http://www.example.com
|
||||
</tt>
|
||||
<br />
|
||||
(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
|
||||
<br />
|
||||
|
||||
|
||||
<br><br>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
|
||||
@@ -118,8 +118,8 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<br>Avoid "never", because the local mirror could be bogus
|
||||
</small><br><br>
|
||||
<!-- -->
|
||||
<li>Parse java files</li>
|
||||
<br><small>Must the engine parse .java files (java classes) to seek included filenames?
|
||||
<li>Parse scripts</li>
|
||||
<br><small>Must the engine parse scripts to seek included filenames?
|
||||
<br>It is checked by default
|
||||
</small><br><br>
|
||||
<!-- -->
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "07 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "12 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -224,7 +224,7 @@ explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
|
||||
.IP \-u
|
||||
check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
|
||||
.IP \-j
|
||||
*parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
|
||||
*parse scripts (j0 don't parse, bitmask: |1 parse default, |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
|
||||
.IP \-sN
|
||||
follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules)) (\-\-robots[=N])
|
||||
.IP \-%h
|
||||
@@ -248,7 +248,7 @@ can also be used to force a specific file type: \-\-assume foo.cgi=text/html
|
||||
.IP \-@iN
|
||||
internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only) (\-\-protocol[=N])
|
||||
.IP \-%w
|
||||
disable a specific external mime module (\-%w htsswf \-%w htsjava) (\-\-disable\-module <param>)
|
||||
disable a specific external mime module (\-%w httrack\-plugin) (\-\-disable\-module <param>)
|
||||
.SS Browser ID:
|
||||
.IP \-F
|
||||
user\-agent field sent in HTTP headers (\-F "user\-agent name") (\-\-user\-agent <param>)
|
||||
@@ -514,8 +514,6 @@ that would have solved them.
|
||||
.SM
|
||||
\- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
|
||||
.SM
|
||||
\- Some java classes may not find some files on them (class included)
|
||||
.SM
|
||||
\- Cgi\-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like \-*cgi\-bin*
|
||||
.SH BUGS
|
||||
Please reports bugs to
|
||||
|
||||
@@ -44,7 +44,7 @@ httrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
|
||||
proxytrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
|
||||
htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
|
||||
|
||||
lib_LTLIBRARIES = libhttrack.la libhtsjava.la
|
||||
lib_LTLIBRARIES = libhttrack.la
|
||||
|
||||
htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
|
||||
proxytrack_SOURCES = proxy/main.c \
|
||||
@@ -82,14 +82,6 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $
|
||||
libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
|
||||
libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
|
||||
|
||||
libhtsjava_la_SOURCES = htsjava.c htsjava.h
|
||||
libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
|
||||
# This thin JNI wrapper reaches libc only through libhttrack, so the direct
|
||||
# libc edge is dropped from DT_NEEDED (library-not-linked-against-libc). Force
|
||||
# libc back as a dependency; configure gates the flag since only a GNU-style
|
||||
# linker accepts it (LIBC_FORCE_LINK is empty on e.g. macOS).
|
||||
libhtsjava_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO) $(LIBC_FORCE_LINK)
|
||||
|
||||
EXTRA_DIST = httrack.h webhttrack \
|
||||
coucal/murmurhash3.h.diff \
|
||||
coucal/murmurhash3.h.orig \
|
||||
@@ -119,7 +111,6 @@ EXTRA_DIST = httrack.h webhttrack \
|
||||
coucal/README.md \
|
||||
coucal/sample.c \
|
||||
coucal/tests.c \
|
||||
htsjava.vcproj \
|
||||
httrack.dsp httrack.dsw httrack.vcproj \
|
||||
libhttrack.dsp libhttrack.dsw libhttrack.vcproj \
|
||||
webhttrack.dsp webhttrack.dsw webhttrack.vcproj
|
||||
|
||||
@@ -3875,7 +3875,6 @@ void voidf(void) {
|
||||
a = "| HTTrack Website Copier |" CRLF;
|
||||
a = "|Code: Windows Interface Xavier Roche |" CRLF;
|
||||
a = "| HTS/HTTrack Xavier Roche |" CRLF;
|
||||
a = "| .class Parser Yann Philippot |" CRLF;
|
||||
a = "| |" CRLF;
|
||||
a = "|Tested on: Windows95,98,NT,2K |" CRLF;
|
||||
a = "| Linux PC |" CRLF;
|
||||
|
||||
@@ -1452,7 +1452,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF
|
||||
("Option %w needs to be followed by a blank space, and a module name");
|
||||
printf("Example: -%%w htsswf\n");
|
||||
printf("Example: -%%w httrack-plugin\n");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
|
||||
@@ -70,7 +70,7 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
|
||||
#endif
|
||||
|
||||
/* Marks a symbol an external wrapper module exports back to the engine.
|
||||
Must override -fvisibility=hidden on ELF, or dlopen()ed plugins (htsjava)
|
||||
Must override -fvisibility=hidden on ELF, or dlopen()ed plugins
|
||||
hide their own hts_plug()/hts_unplug() entry points. */
|
||||
#ifndef EXTERNAL_FUNCTION
|
||||
#ifdef _WIN32
|
||||
|
||||
@@ -213,6 +213,20 @@ const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
|
||||
return strjoker_rec(&memo, chaine, joker, size, size_flag);
|
||||
}
|
||||
|
||||
/* Test-only: strjoker() reporting the work-budget steps spent and the cap, so a
|
||||
self-test can prove the budget bounds a hostile pattern's work. */
|
||||
const char *strjoker_steps(const char *chaine, const char *joker,
|
||||
size_t *nsteps_out, size_t *maxsteps_out) {
|
||||
size_t nsteps = 0;
|
||||
const char *r = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps);
|
||||
|
||||
if (nsteps_out != NULL)
|
||||
*nsteps_out = nsteps;
|
||||
if (maxsteps_out != NULL)
|
||||
*maxsteps_out = STRJOKER_MAXSTEPS;
|
||||
return r;
|
||||
}
|
||||
|
||||
static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
|
||||
const char *joker, LLint *size,
|
||||
int *size_flag) {
|
||||
|
||||
@@ -52,6 +52,10 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
|
||||
oracle for the memoized matcher. */
|
||||
const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
|
||||
int *size_flag);
|
||||
/* strjoker() reporting the work-budget steps it spent and the cap; test-only,
|
||||
lets a self-test assert the budget bounds a hostile pattern's work. */
|
||||
const char *strjoker_steps(const char *chaine, const char *joker,
|
||||
size_t *nsteps_out, size_t *maxsteps_out);
|
||||
const char *strjokerfind(const char *chaine, const char *joker);
|
||||
#endif
|
||||
|
||||
|
||||
@@ -546,8 +546,8 @@ void help(const char *app, int more) {
|
||||
infomsg(" %Y explain which filter rule accepts or rejects a URL, then exit");
|
||||
infomsg
|
||||
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
|
||||
infomsg
|
||||
(" j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
|
||||
infomsg(" j *parse scripts (j0 don't parse, bitmask: |1 parse default, |4 "
|
||||
"don't parse .js |8 don't be aggressive)");
|
||||
infomsg
|
||||
(" sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
|
||||
infomsg
|
||||
@@ -571,8 +571,7 @@ void help(const char *app, int more) {
|
||||
(" can also be used to force a specific file type: --assume foo.cgi=text/html");
|
||||
infomsg
|
||||
(" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
|
||||
infomsg
|
||||
(" %w disable a specific external mime module (-%w htsswf -%w htsjava)");
|
||||
infomsg(" %w disable a specific external mime module (-%w httrack-plugin)");
|
||||
infomsg("");
|
||||
infomsg("Browser ID:");
|
||||
infomsg
|
||||
|
||||
517
src/htsjava.c
517
src/htsjava.c
@@ -1,517 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: Java classes parser */
|
||||
/* Author: Yann Philippot */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
/* Version: Oct/2000 */
|
||||
/* Fixed: problems with class structure (10/2000) */
|
||||
|
||||
// htsjava.c - Parseur de classes java
|
||||
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) )
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
/* Standard httrack module includes */
|
||||
#include "httrack-library.h"
|
||||
#include "htsopt.h"
|
||||
#include "htsdefines.h"
|
||||
|
||||
/* Module structures */
|
||||
#include "htsmodules.h"
|
||||
|
||||
/* We link to libhttrack, we can use its functions */
|
||||
#include "httrack-library.h"
|
||||
|
||||
/* This file */
|
||||
#include "htsjava.h"
|
||||
|
||||
/* calloct/freet wrappers */
|
||||
#include "htssafe.h"
|
||||
|
||||
static int reverse_endian(void) {
|
||||
int endian = 1;
|
||||
|
||||
return (*((char *) &endian) == 1);
|
||||
}
|
||||
|
||||
/* big/little endian swap */
|
||||
#define hts_swap16(A) ( (((A) & 0xFF)<<8) | (((A) & 0xFF00)>>8) )
|
||||
#define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) )
|
||||
|
||||
/* Static definitions */
|
||||
static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT,
|
||||
int *);
|
||||
static unsigned short int readshort(FILE * fp);
|
||||
static int tris(httrackp * opt, char *);
|
||||
static char *printname(char[1024], char[1024]);
|
||||
|
||||
// ** HTS_xx sinon pas pris par VC++
|
||||
#define HTS_CLASS 7
|
||||
#define HTS_FIELDREF 9
|
||||
#define HTS_METHODREF 10
|
||||
#define HTS_STRING 8
|
||||
#define HTS_INTEGER 3
|
||||
#define HTS_FLOAT 4
|
||||
#define HTS_LONG 5
|
||||
#define HTS_DOUBLE 6
|
||||
#define HTS_INTERFACE 11
|
||||
#define HTS_NAMEANDTYPE 12
|
||||
#define HTS_ASCIZ 1
|
||||
#define HTS_UNICODE 2
|
||||
|
||||
#define JAVADEBUG 0
|
||||
|
||||
static const char *libName = "htsjava";
|
||||
|
||||
#ifdef _WIN32
|
||||
#define strcasecmp(a,b) stricmp(a,b)
|
||||
#define strncasecmp(a,b,n) strnicmp(a,b,n)
|
||||
#endif
|
||||
|
||||
static int detect_mime(htsmoduleStruct * str) {
|
||||
const char *savename = str->filename;
|
||||
|
||||
if (savename) {
|
||||
int len = (int) strlen(savename);
|
||||
|
||||
if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
htsmoduleStruct * str) {
|
||||
/* Call parent functions if multiple callbacks are chained. */
|
||||
if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) {
|
||||
if (CALLBACKARG_PREV_FUN(carg, detect)
|
||||
(CALLBACKARG_PREV_CARG(carg), opt, str)) {
|
||||
return 1; /* Found before us, let them have the priority */
|
||||
}
|
||||
}
|
||||
|
||||
/* Check MIME */
|
||||
if (detect_mime(str)) {
|
||||
str->wrapper_name = libName; /* Our ID */
|
||||
return 1; /* Known format, we take it */
|
||||
}
|
||||
|
||||
return 0; /* Unknown format */
|
||||
}
|
||||
|
||||
static off_t fsize(const char *s) {
|
||||
STRUCT_STAT st;
|
||||
|
||||
if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
|
||||
return st.st_size;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
htsmoduleStruct * str) {
|
||||
/* The wrapper_name memebr has changed: not for us anymore */
|
||||
if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) {
|
||||
/* Call parent functions if multiple callbacks are chained. */
|
||||
if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) {
|
||||
return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg),
|
||||
opt, str);
|
||||
}
|
||||
strcpy(str->err_msg,
|
||||
"unexpected error: bad wrapper_name and no previous wrapper");
|
||||
return 0; /* Unexpected error */
|
||||
} else {
|
||||
if (detect_mime(str)) {
|
||||
|
||||
/* (Legacy code) */
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *fpout;
|
||||
JAVA_HEADER header;
|
||||
RESP_STRUCT *tab;
|
||||
const char *file = str->filename;
|
||||
|
||||
str->relativeToHtmlLink = 1;
|
||||
|
||||
#if JAVADEBUG
|
||||
printf("fopen\n");
|
||||
#endif
|
||||
if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
|
||||
NULL) {
|
||||
sprintf(str->err_msg, "Unable to open file %s", file);
|
||||
return 0; // une erreur..
|
||||
}
|
||||
#if JAVADEBUG
|
||||
printf("fread\n");
|
||||
#endif
|
||||
if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
|
||||
fclose(fpout);
|
||||
sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
|
||||
(LLint) fsize(file));
|
||||
return 0;
|
||||
}
|
||||
#if JAVADEBUG
|
||||
printf("header\n");
|
||||
#endif
|
||||
// tester en tête
|
||||
if (reverse_endian()) {
|
||||
header.magic = hts_swap32(header.magic);
|
||||
header.count = hts_swap16(header.count);
|
||||
}
|
||||
if (header.magic != 0xCAFEBABE) {
|
||||
sprintf(str->err_msg, "non java file");
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* A constant-pool entry is >= 1 byte on disk; reject a count exceeding
|
||||
the file size (hostile .class ~68 MB alloc DoS). */
|
||||
if (!hts_count_fits(header.count, (LLint) fsize(file))) {
|
||||
fclose(fpout);
|
||||
sprintf(str->err_msg,
|
||||
"Invalid constant pool count %u (file len " LLintP ")",
|
||||
(unsigned) header.count, (LLint) fsize(file));
|
||||
return 0;
|
||||
}
|
||||
tab = (RESP_STRUCT *) calloct(header.count, sizeof(RESP_STRUCT));
|
||||
if (!tab) {
|
||||
sprintf(str->err_msg, "Unable to alloc %d bytes",
|
||||
(int) sizeof(RESP_STRUCT));
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
}
|
||||
return 0; // erreur..
|
||||
}
|
||||
#if JAVADEBUG
|
||||
printf("calchead\n");
|
||||
#endif
|
||||
{
|
||||
int i;
|
||||
|
||||
for(i = 1; i < header.count; i++) {
|
||||
int err = 0; // ++
|
||||
|
||||
tab[i] = readtable(str, fpout, tab[i], &err);
|
||||
if (!err) {
|
||||
if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE))
|
||||
i++; //2 element si double ou float
|
||||
} else { // ++ une erreur est survenue!
|
||||
if (strnotempty(str->err_msg) == 0)
|
||||
strcpy(str->err_msg, "Internal readtable error");
|
||||
freet(tab);
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#if JAVADEBUG
|
||||
printf("addfiles\n");
|
||||
#endif
|
||||
{
|
||||
unsigned int Class;
|
||||
unsigned int SClass;
|
||||
int i;
|
||||
|
||||
Class = readshort(fpout);
|
||||
SClass = readshort(fpout);
|
||||
|
||||
for(i = 1; i < header.count; i++) {
|
||||
|
||||
if (tab[i].type == HTS_CLASS) {
|
||||
|
||||
if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) {
|
||||
|
||||
if ((tab[i].index1 != SClass) && (tab[i].index1 != Class)
|
||||
&& (tab[tab[i].index1].name[0] != '[')) {
|
||||
|
||||
if (!strstr(tab[tab[i].index1].name, "java/")) {
|
||||
char BIGSTK tempo[1024];
|
||||
|
||||
tempo[0] = '\0';
|
||||
|
||||
snprintf(tempo, sizeof(tempo), "%s.class", tab[tab[i].index1].name);
|
||||
#if JAVADEBUG
|
||||
printf("add %s\n", tempo);
|
||||
#endif
|
||||
if (tab[tab[i].index1].file_position >= 0)
|
||||
str->addLink(str, tempo); /* tab[tab[i].index1].file_position */
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
i = header.count; // exit
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
#if JAVADEBUG
|
||||
printf("end\n");
|
||||
#endif
|
||||
freet(tab);
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
}
|
||||
return 1;
|
||||
|
||||
} else {
|
||||
strcpy(str->err_msg, "bad MIME type");
|
||||
}
|
||||
}
|
||||
return 0; /* Error */
|
||||
}
|
||||
|
||||
/*
|
||||
module entry point
|
||||
*/
|
||||
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
|
||||
EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
|
||||
/* Plug callback functions */
|
||||
CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL);
|
||||
CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL);
|
||||
|
||||
return 1; /* success */
|
||||
}
|
||||
|
||||
// error: !=0 si erreur fatale
|
||||
static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
|
||||
RESP_STRUCT trans, int *error) {
|
||||
char rname[1024];
|
||||
unsigned short int length;
|
||||
int j;
|
||||
|
||||
*error = 0; // pas d'erreur
|
||||
trans.file_position = -1;
|
||||
trans.type = (int) (unsigned char) fgetc(fp);
|
||||
switch (trans.type) {
|
||||
case HTS_CLASS:
|
||||
strcpy(trans.name, "Class");
|
||||
trans.index1 = readshort(fp);
|
||||
break;
|
||||
|
||||
case HTS_FIELDREF:
|
||||
strcpy(trans.name, "Field Reference");
|
||||
trans.index1 = readshort(fp);
|
||||
readshort(fp);
|
||||
break;
|
||||
|
||||
case HTS_METHODREF:
|
||||
strcpy(trans.name, "Method Reference");
|
||||
trans.index1 = readshort(fp);
|
||||
readshort(fp);
|
||||
break;
|
||||
|
||||
case HTS_INTERFACE:
|
||||
strcpy(trans.name, "Interface Method Reference");
|
||||
trans.index1 = readshort(fp);
|
||||
readshort(fp);
|
||||
break;
|
||||
case HTS_NAMEANDTYPE:
|
||||
strcpy(trans.name, "Name and Type");
|
||||
trans.index1 = readshort(fp);
|
||||
readshort(fp);
|
||||
break;
|
||||
|
||||
case HTS_STRING: // CONSTANT_String
|
||||
strcpy(trans.name, "String");
|
||||
trans.index1 = readshort(fp);
|
||||
break;
|
||||
|
||||
case HTS_INTEGER:
|
||||
strcpy(trans.name, "Integer");
|
||||
for(j = 0; j < 4; j++)
|
||||
fgetc(fp);
|
||||
break;
|
||||
|
||||
case HTS_FLOAT:
|
||||
strcpy(trans.name, "Float");
|
||||
for(j = 0; j < 4; j++)
|
||||
fgetc(fp);
|
||||
break;
|
||||
|
||||
case HTS_LONG:
|
||||
strcpy(trans.name, "Long");
|
||||
for(j = 0; j < 8; j++)
|
||||
fgetc(fp);
|
||||
break;
|
||||
case HTS_DOUBLE:
|
||||
strcpy(trans.name, "Double");
|
||||
for(j = 0; j < 8; j++)
|
||||
fgetc(fp);
|
||||
break;
|
||||
|
||||
case HTS_ASCIZ:
|
||||
case HTS_UNICODE:
|
||||
|
||||
if (trans.type == HTS_ASCIZ)
|
||||
strcpy(trans.name, "HTS_ASCIZ");
|
||||
else
|
||||
strcpy(trans.name, "HTS_UNICODE");
|
||||
|
||||
{
|
||||
char BIGSTK buffer[1024];
|
||||
char *p;
|
||||
|
||||
p = &buffer[0];
|
||||
|
||||
trans.file_position = ftell(fp);
|
||||
length = readshort(fp);
|
||||
if (length < HTS_URLMAXSIZE) {
|
||||
while(length > 0) {
|
||||
*p++ = fgetc(fp);
|
||||
|
||||
length--;
|
||||
}
|
||||
*p = '\0';
|
||||
|
||||
if (tris(str->opt, buffer) == 1)
|
||||
str->addLink(str, buffer); /* trans.file_position */
|
||||
else if (tris(str->opt, buffer) == 2)
|
||||
str->addLink(str, printname(rname, buffer));
|
||||
|
||||
strcpy(trans.name, buffer);
|
||||
} else { // gros pb
|
||||
while((length > 0) && (!feof(fp))) {
|
||||
fgetc(fp);
|
||||
length--;
|
||||
}
|
||||
if (!feof(fp)) {
|
||||
trans.type = -1;
|
||||
} else {
|
||||
sprintf(str->err_msg, "Internal structure error (ASCII)");
|
||||
*error = 1;
|
||||
}
|
||||
return (trans);
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
// on arrête tout
|
||||
sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
|
||||
*error = 1;
|
||||
return (trans);
|
||||
break;
|
||||
}
|
||||
return (trans);
|
||||
}
|
||||
|
||||
static unsigned short int readshort(FILE * fp) {
|
||||
unsigned short int valint;
|
||||
|
||||
if (fread(&valint, sizeof(valint), 1, fp) == 1) {
|
||||
if (reverse_endian())
|
||||
return hts_swap16(valint);
|
||||
else
|
||||
return valint;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static int tris(httrackp * opt, char *buffer) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
|
||||
//
|
||||
// Java
|
||||
if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/")))
|
||||
return 2;
|
||||
if (strstr(buffer, ".gif") || strstr(buffer, ".jpg")
|
||||
|| strstr(buffer, ".jpeg") || strstr(buffer, ".au"))
|
||||
return 1;
|
||||
// Ajouts R.X: test type
|
||||
// Autres fichiers
|
||||
{
|
||||
char type[256];
|
||||
|
||||
if (get_httptype_sized(opt, type, sizeof(type), buffer,
|
||||
0)) // recognized type
|
||||
return 1;
|
||||
// ajout RX 05/2001
|
||||
else if (is_dyntype(get_ext(catbuff, sizeof(catbuff), buffer))) // asp,cgi...
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static char *printname(char rname[1024], char name[1024]) {
|
||||
char *p;
|
||||
char *p1;
|
||||
size_t j;
|
||||
|
||||
rname[0] = '\0';
|
||||
//
|
||||
|
||||
p = &name[0];
|
||||
|
||||
if (*p != '[') {
|
||||
return rname; // ""
|
||||
}
|
||||
p += 2;
|
||||
p1 = rname;
|
||||
for(j = 0; name[j] != '\0'; j++, p++) {
|
||||
if (*p == '/')
|
||||
*p1 = '.';
|
||||
if (*p == ';') {
|
||||
*p1 = '\0';
|
||||
strcat(rname, ".class");
|
||||
return rname;
|
||||
} else
|
||||
*p1 = *p;
|
||||
p1++;
|
||||
}
|
||||
p1 -= 3;
|
||||
*p1 = '\0';
|
||||
return rname;
|
||||
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: Java classes parser .h */
|
||||
/* Author: Yann Philippot */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSJAVA_DEFH
|
||||
#define HTSJAVA_DEFH
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
|
||||
#define HTS_DEF_FWSTRUCT_JAVA_HEADER
|
||||
typedef struct JAVA_HEADER JAVA_HEADER;
|
||||
#endif
|
||||
/* 10-byte on-disk .class header image, fread() directly: fields need exact
|
||||
widths (LP64's 8-byte 'unsigned long' magic never matched 0xCAFEBABE). */
|
||||
struct JAVA_HEADER {
|
||||
uint32_t magic;
|
||||
uint16_t minor;
|
||||
uint16_t major;
|
||||
uint16_t count;
|
||||
};
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT
|
||||
#define HTS_DEF_FWSTRUCT_RESP_STRUCT
|
||||
typedef struct RESP_STRUCT RESP_STRUCT;
|
||||
#endif
|
||||
struct RESP_STRUCT {
|
||||
int file_position;
|
||||
//
|
||||
unsigned int index1;
|
||||
unsigned int type;
|
||||
char name[1024];
|
||||
};
|
||||
|
||||
/* Library internal definictions */
|
||||
#ifdef HTS_INTERNAL_BYTECODE
|
||||
|
||||
EXTERNAL_FUNCTION int hts_plug_java(httrackp * opt, const char *argv);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -1,373 +0,0 @@
|
||||
<?xml version="1.0" encoding="Windows-1252"?>
|
||||
<VisualStudioProject
|
||||
ProjectType="Visual C++"
|
||||
Version="9,00"
|
||||
Name="htsjava"
|
||||
ProjectGUID="{2F353408-2C6C-4D31-9AC0-08BDC064DF64}"
|
||||
RootNamespace="htsjava"
|
||||
Keyword="Win32Proj"
|
||||
TargetFrameworkVersion="131072"
|
||||
>
|
||||
<Platforms>
|
||||
<Platform
|
||||
Name="Win32"
|
||||
/>
|
||||
<Platform
|
||||
Name="x64"
|
||||
/>
|
||||
</Platforms>
|
||||
<ToolFiles>
|
||||
</ToolFiles>
|
||||
<Configurations>
|
||||
<Configuration
|
||||
Name="Debug|Win32"
|
||||
OutputDirectory="C:\temp\htsjava\Debug"
|
||||
IntermediateDirectory="C:\temp\htsjava\Debug"
|
||||
ConfigurationType="2"
|
||||
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
|
||||
PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="1"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="3"
|
||||
Detect64BitPortabilityProblems="true"
|
||||
DebugInformationFormat="4"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
OutputFile="C:\temp\httrack\htsjava.dll"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
|
||||
SubSystem="2"
|
||||
RandomizedBaseAddress="1"
|
||||
DataExecutionPrevention="0"
|
||||
ImportLibrary="$(OutDir)/htsjava.lib"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Debug|x64"
|
||||
OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="2"
|
||||
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
|
||||
PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="1"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="3"
|
||||
Detect64BitPortabilityProblems="true"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
OutputFile="C:\temp\httrack64\htsjava.dll"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
|
||||
SubSystem="2"
|
||||
RandomizedBaseAddress="1"
|
||||
DataExecutionPrevention="0"
|
||||
ImportLibrary="$(OutDir)/htsjava.lib"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|Win32"
|
||||
OutputDirectory="C:\temp\htsjava\Release"
|
||||
IntermediateDirectory="C:\temp\htsjava\Release"
|
||||
ConfigurationType="2"
|
||||
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
|
||||
PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
|
||||
RuntimeLibrary="0"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="3"
|
||||
Detect64BitPortabilityProblems="true"
|
||||
DebugInformationFormat="0"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
OutputFile="O:\HTTrack\httrack\htsjava.dll"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
ProgramDatabaseFile="O:\HTTrack\httrack\htsjava.pdb"
|
||||
SubSystem="2"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
RandomizedBaseAddress="1"
|
||||
DataExecutionPrevention="0"
|
||||
ImportLibrary="$(OutDir)/htsjava.lib"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
CommandLine=""C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe" sign /f "Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx" /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)
"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|x64"
|
||||
OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="2"
|
||||
InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
|
||||
PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
|
||||
RuntimeLibrary="0"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="3"
|
||||
Detect64BitPortabilityProblems="true"
|
||||
DebugInformationFormat="0"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
OutputFile="O:\HTTrack\httrack\x64\htsjava.dll"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
ProgramDatabaseFile="O:\HTTrack\httrack\x64\htsjava.pdb"
|
||||
SubSystem="2"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
RandomizedBaseAddress="1"
|
||||
DataExecutionPrevention="0"
|
||||
ImportLibrary="$(OutDir)/htsjava.lib"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
CommandLine=""C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe" sign /f "Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx" /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)
"
|
||||
/>
|
||||
</Configuration>
|
||||
</Configurations>
|
||||
<References>
|
||||
</References>
|
||||
<Files>
|
||||
<Filter
|
||||
Name="Source Files"
|
||||
Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="htsjava.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
Filter="h;hpp;hxx;hm;inl;inc;xsd"
|
||||
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
|
||||
>
|
||||
<File
|
||||
RelativePath="htsjava.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Resource Files"
|
||||
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
|
||||
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
|
||||
>
|
||||
</Filter>
|
||||
</Files>
|
||||
<Globals>
|
||||
</Globals>
|
||||
</VisualStudioProject>
|
||||
12
src/htslib.c
12
src/htslib.c
@@ -5926,17 +5926,7 @@ int multipleStringMatch(const char *s, const char *match) {
|
||||
}
|
||||
|
||||
HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
#if ( defined(_WIN32) || defined(__ANDROID__) )
|
||||
static const char *defaultModules[] = {
|
||||
"htsswf", "htsjava", "httrack-plugin", NULL
|
||||
};
|
||||
#else
|
||||
#ifndef HTS_LIBHTSJAVA_NAME
|
||||
#define HTS_LIBHTSJAVA_NAME "libhtsjava.so" /* non-autoconf fallback */
|
||||
#endif
|
||||
static const char *defaultModules[] = {"libhtsswf.so.1", HTS_LIBHTSJAVA_NAME,
|
||||
"httrack-plugin", NULL};
|
||||
#endif
|
||||
static const char *defaultModules[] = {"httrack-plugin", NULL};
|
||||
httrackp *opt = malloc(sizeof(httrackp));
|
||||
|
||||
/* default options */
|
||||
|
||||
@@ -456,13 +456,6 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
|
||||
return b->buf;
|
||||
}
|
||||
|
||||
/** True if 'count' records of >= 1 byte each fit in 'available' bytes; guards
|
||||
an attacker-controlled count driving a large allocation. */
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean hts_count_fits(size_t count,
|
||||
LLint available) {
|
||||
return (available >= 0 && (LLint) count <= available) ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
|
||||
added bounds checking. freet() also NULLs the freed pointer and tolerates
|
||||
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the
|
||||
|
||||
@@ -744,29 +744,33 @@ static int st_filterdual(httrackp *opt, int argc, char **argv) {
|
||||
process (OSS-Fuzz 5060751291908096 / 5745936014573568). */
|
||||
static int st_filterbounds(httrackp *opt, int argc, char **argv) {
|
||||
const size_t big = 100000; /* well past the length cap */
|
||||
const size_t stars = 900; /* star-heavy pattern, under the cap */
|
||||
const size_t stars = 1023; /* pattern len 2047, under the length cap */
|
||||
const size_t subjlen = 2048;
|
||||
char *subj = malloct(big + 1);
|
||||
char *pat = malloct(2 * stars + 2);
|
||||
size_t i;
|
||||
size_t steps = 0, maxsteps = 0, i;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
memset(subj, 'a', big);
|
||||
subj[big] = '\0';
|
||||
/* '*' matches anything, but an over-length subject is refused by the cap */
|
||||
/* '*' matches anything, but an over-length subject trips the length cap */
|
||||
assertf(strjoker(subj, "*", NULL, NULL) == NULL);
|
||||
assertf(strjokerfind(subj, "*") == NULL);
|
||||
/* within-cap star-heavy dead-end: the step budget keeps it bounded (a hang
|
||||
would time the test out) */
|
||||
/* Star-heavy dead-end at the length cap: unbounded it runs ~1.26e9 memo-steps
|
||||
(~6s). */
|
||||
for (i = 0; i < stars; i++) {
|
||||
pat[2 * i] = '*';
|
||||
pat[2 * i + 1] = 'a';
|
||||
}
|
||||
pat[2 * stars] = 'b'; /* never matches an all-'a' subject */
|
||||
pat[2 * stars + 1] = '\0';
|
||||
subj[stars] = '\0';
|
||||
assertf(strjoker(subj, pat, NULL, NULL) == NULL);
|
||||
subj[subjlen] = '\0';
|
||||
/* Budget must fire and hold: steps > cap (deleting the budget zeroes the
|
||||
counter that is the enforcement), steps < 10*cap (unbudgeted ~1.26e9). */
|
||||
assertf(strjoker_steps(subj, pat, &steps, &maxsteps) == NULL);
|
||||
assertf(steps > maxsteps && steps < 10 * maxsteps);
|
||||
assertf(strjokerfind(subj, pat) == NULL);
|
||||
freet(pat);
|
||||
freet(subj);
|
||||
@@ -2580,21 +2584,6 @@ static int st_ftpuser(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* hts_count_fits caps the .class constant-pool entry count to the file size,
|
||||
rejecting the ~68 MB-per-file calloc DoS. */
|
||||
static int st_java(httrackp *opt, int argc, char **argv) {
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
assertf(hts_count_fits(10, 1000) == HTS_TRUE);
|
||||
assertf(hts_count_fits(0, 10) == HTS_TRUE);
|
||||
assertf(hts_count_fits(65535, 10) == HTS_FALSE);
|
||||
assertf(hts_count_fits(1, 0) == HTS_FALSE);
|
||||
assertf(hts_count_fits(1, -1) == HTS_FALSE);
|
||||
printf("java constant-pool cap self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* Registry: name -> handler, with a usage hint and a one-line description. */
|
||||
/* ------------------------------------------------------------ */
|
||||
@@ -2695,7 +2684,6 @@ static const struct selftest_entry {
|
||||
st_ftpline},
|
||||
#endif
|
||||
{"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
|
||||
{"java", "", "java .class constant-pool count cap self-test", st_java},
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
@@ -224,14 +224,6 @@ SOURCE=htsinthash.c
|
||||
# End Source File
|
||||
# Begin Source File
|
||||
|
||||
SOURCE=htsjava.c
|
||||
# End Source File
|
||||
# Begin Source File
|
||||
|
||||
SOURCE=htsjava.h
|
||||
# End Source File
|
||||
# Begin Source File
|
||||
|
||||
SOURCE=htslib.c
|
||||
# End Source File
|
||||
# Begin Source File
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# .class constant-pool count is capped to the file size (calloc DoS).
|
||||
httrack -O /dev/null -#test=java run | grep -q "java constant-pool cap self-test OK"
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
# The java plugin must load (versioned dlopen name) and parse a .class
|
||||
# constant pool: a resource named only inside Foo.class gets crawled.
|
||||
set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
tmproot=$(mktemp -d)
|
||||
trap 'rm -rf "$tmproot"' EXIT
|
||||
mkdir "$tmproot/javaclass"
|
||||
|
||||
cat >"$tmproot/javaclass/index.html" <<'EOF'
|
||||
<html><body><a href="Foo.class">applet</a></body></html>
|
||||
EOF
|
||||
printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
|
||||
# magic/minor/major, count=2, one CONSTANT_Utf8 "hello.gif", class/superclass
|
||||
printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
|
||||
>"$tmproot/javaclass/Foo.class"
|
||||
|
||||
# --log-found first: a plugin-load failure (wrong soname, shadowing system
|
||||
# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
|
||||
--log-found '\+libhtsjava' \
|
||||
--found 'javaclass/Foo.class' \
|
||||
--found 'javaclass/hello.gif' \
|
||||
httrack 'BASEURL/javaclass/index.html'
|
||||
@@ -47,7 +47,6 @@ TESTS = \
|
||||
01_engine-identabs.test \
|
||||
01_engine-escape-room.test \
|
||||
01_engine-inplace-escape.test \
|
||||
01_engine-java.test \
|
||||
01_engine-makeindex.test \
|
||||
01_engine-mime.test \
|
||||
01_engine-parse.test \
|
||||
@@ -103,7 +102,6 @@ TESTS = \
|
||||
28_local-pause.test \
|
||||
29_local-redirect-fragment.test \
|
||||
30_local-fragment-link.test \
|
||||
31_local-javaclass.test \
|
||||
32_local-cdispo.test \
|
||||
33_local-delayed.test \
|
||||
34_local-maxtime.test \
|
||||
|
||||
Reference in New Issue
Block a user