./htscore.c
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998-2015 Xavier Roche and other contributors
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Important notes:
- We hereby ask people using this source NOT to use it in purpose of grabbing
emails addresses, or collecting any other private information on persons.
This would disgrace our work, and spoil the many hours we spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: Main source */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
/* Internal engine bytecode */
#define HTS_INTERNAL_BYTECODE
#include <fcntl.h>
#include <ctype.h>
/* File defs */
#include "htscore.h"
/* specific definitions */
#include "htsbase.h"
#include "htsnet.h"
#include "htsbauth.h"
#include "htsmd5.h"
#include "htsindex.h"
/* external modules */
#include "htsmodules.h"
// htswrap_add
#include "htswrap.h"
// parser
#include "htsparse.h"
/* Cache */
#include "htszlib.h"
/* Charset handling */
#include "htscharset.h"
/* Dynamic typed arrays */
#include "htsarrays.h"
/* END specific definitions */
/* external modules */
extern int hts_parse_externals(htsmoduleStruct * str);
extern void htspe_init(void);
/* debug */
#if DEBUG_SHOWTYPES
char REG[32768] = "\n";
#endif
#if NSDEBUG
int nsocDEBUG = 0;
#endif
//
#define _CLRSCR printf("\33[m\33[2J");
#define _GOTOXY(X,Y) printf("\33[" X ";" Y "f");
#if DEBUG_CHECKINT
#define _CHECKINT_FAIL(a) printf("\n%s\n",a); fflush(stdout); abort();
#define _CHECKINT(obj_ptr,message) \
if (obj_ptr) {\
if (( * ((char*) (obj_ptr)) != 0) || ( * ((char*) (((char*) (obj_ptr)) + sizeof(*(obj_ptr))-1)) != 0)) {\
char msg[1100];\
if (( * ((char*) (obj_ptr)) != 0) && ( * ((char*) (((char*) (obj_ptr)) + sizeof(*(obj_ptr))-1)) != 0))\
sprintf(msg,"* PANIC: Integrity error (structure crushed) in: %s",message);\
else if ( * ((char*) (obj_ptr)) != 0)\
sprintf(msg,"* PANIC: Integrity error (start of structure) in: %s",message);\
else\
sprintf(msg,"* PANIC: Integrity error (end of structure) in: %s",message);\
_CHECKINT_FAIL(msg);\
}\
} else {\
char msg[1100];\
sprintf(msg,"* PANIC: NULL pointer in: %s",message);\
_CHECKINT_FAIL(msg);\
}
#endif
#if DEBUG_HASH
// longest hash chain?
int longest_hash[3] = { 0, 0, 0 }, hashnumber = 0;
#endif
// Début de httpmirror, routines annexes
// au cas où nous devons quitter rapidement xhttpmirror (plus de mémoire, etc)
// note: partir de liens_max.. vers 0.. sinon erreur de violation de mémoire: les liens suivants
// ne sont plus à nous.. agh! [dur celui-là]
#define HTMLCHECK_UNINIT { \
hts_log_print(opt, LOG_INFO, "engine: end"); \
RUN_CALLBACK0(opt, end); \
}
#define XH_extuninit do { \
HTMLCHECK_UNINIT \
hts_record_free(opt); \
if (filters && filters[0]) { \
freet(filters[0]); filters[0]=NULL; \
} \
if (filters) { \
freet(filters); filters=NULL; \
} \
back_delete_all(opt,&cache,sback); \
back_free(&sback); \
checkrobots_free(&robots);\
if (cache.use) { freet(cache.use); cache.use=NULL; } \
if (cache.dat) { fclose(cache.dat); cache.dat=NULL; } \
if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
if (cache.zipOutput) { \
zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
cache.zipOutput = NULL; \
} \
if (cache.zipInput) { \
unzClose(cache.zipInput); \
cache.zipInput = NULL; \
} \
if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
if (opt->log != NULL) fflush(opt->log); \
if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
if (cache_tests) { coucal_delete(&cache_tests); } \
if (template_header) { freet(template_header); template_header=NULL; } \
if (template_body) { freet(template_body); template_body=NULL; } \
if (template_footer) { freet(template_footer); template_footer=NULL; } \
hash_free(&hash); \
clearCallbacks(&opt->state.callbacks); \
/*structcheck_init(-1);*/ \
} while(0)
#define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)
struct lien_buffers {
/* Main array of pointers.
This is the real "lien_url **liens" pointer base. */
TypedArray(lien_url*) ptr;
/* String pool, chunked. */
char *string_buffer;
size_t string_buffer_size;
size_t string_buffer_capa;
TypedArray(char*) string_buffers;
/* Structure list, chunked. */
lien_url *lien_buffer;
size_t lien_buffer_size;
size_t lien_buffer_capa;
TypedArray(lien_url*) lien_buffers;
};
// duplicate a string, or return NULL upon error (out-of-memory)
static char* hts_record_link_strdup_(httrackp *opt, const char *s) {
static const size_t block_capa = 32768;
lien_buffers *const liensbuf = opt->liensbuf;
const size_t len = strlen(s) + 1; /* including terminating \0 */
char *s_dup;
assertf(liensbuf != NULL);
assertf(len < block_capa);
// not enough capacity ? then create a new chunk
if (len + liensbuf->string_buffer_size > liensbuf->string_buffer_capa) {
// backup current block pointer for later free
if (liensbuf->string_buffer != NULL) {
TypedArrayAdd(liensbuf->string_buffers, liensbuf->string_buffer);
liensbuf->string_buffer = NULL;
liensbuf->string_buffer_size = 0;
}
// Double capacity for each new chained block
liensbuf->string_buffer_capa =
liensbuf->string_buffer_capa < block_capa
? block_capa : liensbuf->string_buffer_capa * 2;
liensbuf->string_buffer = malloct(liensbuf->string_buffer_capa);
if (liensbuf->string_buffer == NULL) {
hts_record_assert_memory_failed(liensbuf->string_buffer_capa);
}
liensbuf->string_buffer_size = 0;
hts_log_print(opt, LOG_DEBUG,
"reallocated %d new bytes of strings room",
(int) liensbuf->string_buffer_capa);
}
assertf(len + liensbuf->string_buffer_size <= liensbuf->string_buffer_capa);
s_dup = &liensbuf->string_buffer[liensbuf->string_buffer_size];
memcpy(s_dup, s, len);
liensbuf->string_buffer_size += len;
return s_dup;
}
static char* hts_record_link_strdup(httrackp *opt, const char *s) {
assertf(opt != NULL);
assertf(s != NULL);
return hts_record_link_strdup_(opt, s);
}
size_t hts_record_link_latest(httrackp *opt) {
lien_buffers *const liensbuf = opt->liensbuf;
assertf(TypedArraySize(liensbuf->ptr) != 0);
return TypedArraySize(liensbuf->ptr) - 1;
}
// returns a new zeroed lien_url entry,
// or (size_t) -1 upon error (out-of-memory)
// the returned index is the osset within opt->liens[]
static size_t hts_record_link_alloc(httrackp *opt) {
static const size_t block_capa = 256;
lien_buffers *const liensbuf = opt->liensbuf;
lien_url *link;
assertf(opt != NULL);
assertf(liensbuf != NULL);
// Limit the number of links
if (opt->maxlink > 0 && TypedArraySize(liensbuf->ptr) >= opt->maxlink) {
return (size_t) -1;
}
// Create a new chunk of lien_url[]
// There are references to item pointers, so we can not just realloc()
if (liensbuf->lien_buffer_size == liensbuf->lien_buffer_capa) {
size_t capa_bytes;
if (liensbuf->lien_buffer != NULL) {
TypedArrayAdd(liensbuf->lien_buffers, liensbuf->lien_buffer);
liensbuf->lien_buffer_size = 0;
}
// Double capacity for each new chained block
liensbuf->lien_buffer_capa =
liensbuf->lien_buffer_capa < block_capa
? block_capa : liensbuf->lien_buffer_capa * 2;
capa_bytes = liensbuf->lien_buffer_capa*sizeof(*liensbuf->lien_buffer);
liensbuf->lien_buffer = (lien_url*) malloct(capa_bytes);
if (liensbuf->lien_buffer == NULL) {
hts_record_assert_memory_failed(capa_bytes);
}
liensbuf->lien_buffer_size = 0;
hts_log_print(opt, LOG_DEBUG, "reallocated %d new link placeholders",
(int) liensbuf->lien_buffer_capa);
}
// Take next lien_url item
assertf(liensbuf->lien_buffer_size < liensbuf->lien_buffer_capa);
link = &liensbuf->lien_buffer[liensbuf->lien_buffer_size++];
memset(link, 0, sizeof(*link));
// Add new lien_url pointer to the array of links
TypedArrayAdd(liensbuf->ptr, link);
// Ensure we have a guard NULL
TypedArrayAdd(liensbuf->ptr, NULL);
TypedArraySize(liensbuf->ptr)--;
// Update pointer as it may have changed,
// and update heap top index
opt->liens = TypedArrayElts(liensbuf->ptr);
assertf(TypedArraySize(liensbuf->ptr) != 0);
assertf(TypedArraySize(liensbuf->ptr) < ( (unsigned int) -1 ) / 2);
opt->lien_tot = (int) TypedArraySize(liensbuf->ptr);
// return tail
return hts_record_link_latest(opt);
}
void hts_record_init(httrackp *opt) {
if (opt->liensbuf == NULL) {
opt->liensbuf = calloct(sizeof(*opt->liensbuf), 1);
if (opt->liensbuf == NULL) {
hts_record_assert_memory_failed(sizeof(*opt->liensbuf));
}
}
}
// wipe records
void hts_record_free(httrackp *opt) {
lien_buffers *const liensbuf = opt->liensbuf;
if (liensbuf != NULL) {
size_t i;
TypedArrayFree(liensbuf->ptr);
if (liensbuf->string_buffer != NULL) {
freet(liensbuf->string_buffer);
liensbuf->string_buffer = NULL;
liensbuf->string_buffer_size = 0;
liensbuf->string_buffer_capa = 0;
}
for(i = 0 ; i < TypedArraySize(liensbuf->string_buffers) ; i++) {
freet(TypedArrayNth(liensbuf->string_buffers, i));
TypedArrayNth(liensbuf->string_buffers, i) = NULL;
}
TypedArrayFree(liensbuf->string_buffers);
if (liensbuf->lien_buffer != NULL) {
freet(liensbuf->lien_buffer);
liensbuf->lien_buffer = NULL;
}
for(i = 0 ; i < TypedArraySize(liensbuf->lien_buffers) ; i++) {
freet(TypedArrayNth(liensbuf->lien_buffers, i));
TypedArrayNth(liensbuf->lien_buffers, i) = NULL;
}
TypedArrayFree(liensbuf->lien_buffers);
freet(opt->liensbuf);
opt->liensbuf = NULL;
}
opt->liens = NULL; // no longer defined
}
// adds a new link and returns a non-zero value upon success
static int hts_record_link_(httrackp * opt,
const char *address, const char *file, const char *save,
const char *ref_address, const char *ref_file,
const char *codebase) {
// create a new entry
const size_t lien_tot = hts_record_link_alloc(opt);
lien_url*const link = lien_tot != (size_t) -1 ? opt->liens[lien_tot] : NULL;
if (link == NULL) {
return 0;
}
// record string fields
if ( (link->adr = hts_record_link_strdup(opt, address)) == NULL
|| (link->fil = hts_record_link_strdup(opt, file)) == NULL
|| (link->sav = hts_record_link_strdup(opt, save)) == NULL
|| (link->former_adr = hts_record_link_strdup(opt, ref_address)) == NULL
|| (link->former_fil = hts_record_link_strdup(opt, ref_file)) == NULL
) {
return 0;
}
// record codebase for java classes
if (codebase != NULL) {
const size_t len = strlen(file);
if (len > 6 && strncmp(&file[len - 6], ".class", 6) == 0) {
if ((link->cod = hts_record_link_strdup(opt, codebase)) == NULL) {
return 0;
}
}
}
// add entry in the hashtables
hash_write(opt->hash, lien_tot);
// success
return 1;
}
int hts_record_link(httrackp * opt,
const char *address, const char *file, const char *save,
const char *ref_address, const char *ref_file,
const char *codebase) {
const int success =
hts_record_link_(opt, address, file, save, ref_address, ref_file, codebase);
if (!success) {
hts_log_print(opt, LOG_PANIC, "Too many links (links=%ld, limit=%ld)",
(long int) heap_top_index(), (long int) opt->maxlink);
hts_log_print(opt, LOG_INFO,
"To avoid that: use #L option for more links (example: -#L1000000, or -#L0 to disable)");
}
return success;
}
void hts_invalidate_link(httrackp * opt, int lpos) {
/* devalidate entry */
opt->liens[lpos]->pass2 = -1;
}
#define HT_INDEX_END do { \
if (!makeindex_done) { \
if (makeindex_fp) { \
char BIGSTK tempo[1024]; \
if (makeindex_links == 1) { \
char BIGSTK link_escaped[HTS_URLMAXSIZE*2]; \
escape_uri_utf(makeindex_firstlink, link_escaped, sizeof(link_escaped)); \
sprintf(tempo,"<meta HTTP-EQUIV=\"Refresh\" CONTENT=\"0; URL=%s\">"CRLF, link_escaped); \
} else \
tempo[0]='\0'; \
hts_template_format(makeindex_fp,template_footer, \
"<!-- Mirror and index made by HTTrack Website Copier/"HTTRACK_VERSION" "HTTRACK_AFF_AUTHORS" -->", \
tempo, /* EOF */ NULL \
); \
fflush(makeindex_fp); \
fclose(makeindex_fp); /* à ne pas oublier sinon on passe une nuit blanche */ \
makeindex_fp=NULL; \
usercommand(opt,0,NULL,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_html_utf8),"index.html"),"",""); \
} \
} \
makeindex_done=1; /* ok c'est fait */ \
} while(0)
/* does it look like XML ? (SVG et al.) */
static int look_like_xml(const char *s) {
return strncmp(s, "<?xml", 5) == 0
|| strncmp(s, "<!-- ", 5) == 0
|| strncmp(s, "<svg ", 5) == 0
;
}
// Début de httpmirror, robot
// url1 peut être multiple
int httpmirror(char *url1, httrackp * opt) {
char *primary = NULL; // première page, contenant les liens à scanner
hash_struct hash; // système de hachage, accélère la recherche dans les liens
hash_struct *const hashptr = &hash;
t_cookie BIGSTK cookie; // gestion des cookies
//char* tab_alloc=NULL;
int ptr; // pointeur actuel sur les liens
//
int numero_passe = 0; // deux passes pour html puis images
struct_back *sback = NULL;
htsblk BIGSTK r; // retour de certaines fonctions
// pour les stats, nombre de fichiers & octets écrits
LLint stat_fragment = 0; // pour la fragmentation
//TStamp istat_timestart; // départ pour calcul instantanné
//
TStamp last_info_shell = 0;
int info_shell = 0;
// filtres
char **filters = NULL;
//int filter_max=0;
int filptr = 0;
//
int makeindex_done = 0; // lorsque l'index sera fait
FILE *makeindex_fp = NULL;
int makeindex_links = 0;
char BIGSTK makeindex_firstlink[HTS_URLMAXSIZE * 2];
// statistiques (mode #Z)
FILE *makestat_fp = NULL; // fichier de stats taux transfert
FILE *maketrack_fp = NULL; // idem pour le tracking
TStamp makestat_time = 0; // attente (secondes)
LLint makestat_total = 0; // repère du nombre d'octets transférés depuis denrière stat
int makestat_lnk = 0; // idem, pour le nombre de liens
//
char BIGSTK codebase[HTS_URLMAXSIZE * 2]; // base pour applet java
char BIGSTK base[HTS_URLMAXSIZE * 2]; // base pour les autres fichiers
//
cache_back BIGSTK cache;
robots_wizard BIGSTK robots; // gestion robots.txt
coucal cache_hashtable = NULL;
coucal cache_tests = NULL;
//
char *template_header = NULL, *template_body = NULL, *template_footer = NULL;
//
codebase[0] = '\0';
base[0] = '\0';
//
cookie.auth.next = NULL;
cookie.auth.auth[0] = cookie.auth.prefix[0] = '\0';
//
// noter heure actuelle de départ en secondes
memset(&HTS_STAT, 0, sizeof(HTS_STAT));
HTS_STAT.stat_timestart = time_local();
//istat_timestart=stat_timestart;
HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
/* reset stats */
HTS_STAT.HTS_TOTAL_RECV = 0;
HTS_STAT.istat_bytes[0] = HTS_STAT.istat_bytes[1] = 0;
if (opt->shell) {
last_info_shell = HTS_STAT.stat_timestart;
}
if ((opt->makestat) || (opt->maketrack)) {
makestat_time = HTS_STAT.stat_timestart;
}
// init external modules
htspe_init();
// initialiser cookie
if (opt->accept_cookie) {
opt->cookie = &cookie;
cookie.max_len = 30000; // max len
strcpybuff(cookie.data, "");
// Charger cookies.txt par défaut ou cookies.txt du miroir
cookie_load(opt->cookie, StringBuff(opt->path_log), "cookies.txt");
cookie_load(opt->cookie, "", "cookies.txt");
} else
opt->cookie = NULL;
// initialiser exit_xh
opt->state.exit_xh = 0; // sortir prématurément (var globale)
// initialiser usercommand
usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");
// initialiser structcheck
// structcheck_init(1);
// initialiser verif_backblue
verif_backblue(opt, NULL);
verif_external(opt, 0, 0);
verif_external(opt, 1, 0);
// et templates html
template_header =
readfile_or(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
"templates/index-header.html"), HTS_INDEX_HEADER);
template_body =
readfile_or(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
"templates/index-body.html"), HTS_INDEX_BODY);
template_footer =
readfile_or(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
"templates/index-footer.html"), HTS_INDEX_FOOTER);
// initialiser mimedefs
//get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
// Initialiser indexation
if (opt->kindex)
index_init(StringBuff(opt->path_html));
// effacer bloc cache
memset(&cache, 0, sizeof(cache_back));
cache.type = opt->cache; // cache?
cache.errlog = cache.log = opt->log; // err log?
cache.ptr_ant = cache.ptr_last = 0; // pointeur pour anticiper
// initialiser hash cache
cache_hashtable = coucal_new(0);
cache_tests = coucal_new(0);
if (cache_hashtable == NULL || cache_tests == NULL) {
printf("PANIC! : Not enough memory [%d]\n", __LINE__);
filters[0] = NULL; // uniquement a cause du warning de XH_extuninit
XH_extuninit;
return 0;
}
hts_set_hash_handler(cache_hashtable, opt);
hts_set_hash_handler(cache_tests, opt);
coucal_set_name(cache_hashtable, "cache_hashtable");
coucal_set_name(cache_tests, "cache_tests");
coucal_value_is_malloc(cache_tests, 1); /* malloc */
cache.hashtable = (void *) cache_hashtable; /* copy backcache hash */
cache.cached_tests = (void *) cache_tests; /* copy of cache_tests */
// robots.txt
strcpybuff(robots.adr, "!"); // dummy
robots.token[0] = '\0';
robots.next = NULL; // suivant
opt->robotsptr = &robots;
// effacer filters
opt->maxfilter = maximum(opt->maxfilter, 128);
if (filters_init(&filters, opt->maxfilter, 0) == 0) {
printf("PANIC! : Not enough memory [%d]\n", __LINE__);
XH_extuninit;
return 0;
}
opt->filters.filters = &filters;
//
opt->filters.filptr = &filptr;
//opt->filters.filter_max=&filter_max;
// hash table
opt->hash = &hash;
// initialize link heap
hts_record_init(opt);
// initialiser ptr et lien_tot
ptr = 0;
// initialiser hachage
hash_init(opt, &hash, opt->urlhack);
// note: we need a cast because of the const
hash.liens = (const lien_url *const*const*) &opt->liens;
// copier adresse(s) dans liste des adresses
{
char *a = url1;
int primary_len = 8192;
if (StringNotEmpty(opt->filelist)) {
primary_len += max(0, fsize(StringBuff(opt->filelist)) * 2);
}
primary_len += (int) strlen(url1) * 2;
// création de la première page, qui contient les liens de base à scanner
// c'est plus propre et plus logique que d'entrer à la main les liens dans la pile
// on bénéficie ainsi des vérifications et des tests du robot pour les liens "primaires"
primary = (char *) malloct(primary_len);
if (primary) {
primary[0] = '\0';
} else {
printf("PANIC! : Not enough memory [%d]\n", __LINE__);
XH_extuninit;
return 0;
}
while(*a) {
int i;
int joker = 0;
// vérifier qu'il n'y a pas de * dans l'url
if (*a == '+')
joker = 1;
else if (*a == '-')
joker = 1;
if (joker) { // joker ou filters
//char* p;
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
int type;
int plus = 0;
// noter joker (dans b)
if (*a == '+') { // champ +
type = 1;
plus = 1;
a++;
} else if (*a == '-') { // champ forbidden[]
type = 0;
a++;
} else { // champ + avec joker sans doute
type = 1;
}
// recopier prochaine chaine (+ ou -)
i = 0;
while((*a != 0) && (!isspace((unsigned char) *a))) {
tempo[i++] = *a;
a++;
}
tempo[i++] = '\0';
while(isspace((unsigned char) *a)) {
a++;
}
// sauter les + sans rien après..
if (strnotempty(tempo)) {
if ((plus == 0) && (type == 1)) { // implicite: *www.edf.fr par exemple
if (tempo[strlen(tempo) - 1] != '*') {
strcatbuff(tempo, "*"); // ajouter un *
}
}
if (type)
strcpybuff(filters[filptr], "+");
else
strcpybuff(filters[filptr], "-");
strcatbuff(filters[filptr], tempo);
filptr++;
/* sanity check */
if (filptr + 1 >= opt->maxfilter) {
opt->maxfilter += HTS_FILTERSINC;
if (filters_init(&filters, opt->maxfilter, HTS_FILTERSINC) == 0) {
printf("PANIC! : Too many filters : >%d [%d]\n", filptr,
__LINE__);
hts_log_print(opt, LOG_PANIC,
"Too many filters, giving up..(>%d)", filptr);
hts_log_print(opt, LOG_NOTICE,
"To avoid that: use #F option for more filters (example: -#F5000)");
XH_extuninit;
return 0;
}
//opt->filters.filters=filters;
}
}
} else { // adresse normale
char BIGSTK url[HTS_URLMAXSIZE * 2];
// prochaine adresse
i = 0;
while((*a != 0) && (!isspace((unsigned char) *a))) {
url[i++] = *a;
a++;
}
while(isspace((unsigned char) *a)) {
a++;
}
url[i++] = '\0';
//strcatbuff(primary,"<PRIMARY=\"");
if (strstr(url, ":/") == NULL)
strcatbuff(primary, "http://");
strcatbuff(primary, url);
//strcatbuff(primary,"\">");
strcatbuff(primary, "\n");
}
} // while
/* load URL file list */
/* OPTIMIZED for fast load */
if (StringNotEmpty(opt->filelist)) {
char *filelist_buff = NULL;
const size_t filelist_sz = off_t_to_size_t(fsize(StringBuff(opt->filelist)));
if (filelist_sz != (size_t) -1) {
FILE *fp = fopen(StringBuff(opt->filelist), "rb");
if (fp) {
filelist_buff = malloct(filelist_sz + 1);
if (filelist_buff) {
if (fread(filelist_buff, 1, filelist_sz, fp) != filelist_sz) {
freet(filelist_buff);
filelist_buff = NULL;
} else {
*(filelist_buff + filelist_sz) = '\0';
}
}
fclose(fp);
}
}
if (filelist_buff) {
int filelist_ptr = 0;
int n = 0;
char BIGSTK line[HTS_URLMAXSIZE * 2];
char *primary_ptr = primary + strlen(primary);
while(filelist_ptr < filelist_sz) {
int count =
binput(filelist_buff + filelist_ptr, line, HTS_URLMAXSIZE);
filelist_ptr += count;
if (count && line[0]) {
n++;
if (strstr(line, ":/") == NULL) {
strcpybuff(primary_ptr, "http://");
primary_ptr += strlen(primary_ptr);
}
strcpybuff(primary_ptr, line);
primary_ptr += strlen(primary_ptr);
strcpybuff(primary_ptr, "\n");
primary_ptr += 1;
}
}
// fclose(fp);
hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
StringBuff(opt->filelist));
// Free buffer
freet(filelist_buff);
} else {
hts_log_print(opt, LOG_ERROR, "Could not include URL list: %s",
StringBuff(opt->filelist));
}
}
// lien primaire
if (!hts_record_link(opt, "primary", "/primary",
fslash(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_html_utf8), "index.html")),
"", "", NULL)) {
XH_extuninit; // désallocation mémoire & buffers
return 0;
}
heap_top()->testmode = 0; // pas mode test
heap_top()->link_import = 0; // pas mode import
heap_top()->depth = opt->depth + 1; // lien de priorité maximale
heap_top()->pass2 = 0; // 1ère passe
heap_top()->retry = opt->retry; // lien de priorité maximale
heap_top()->premier = heap_top_index(); // premier lien, objet-père=objet
heap_top()->precedent = heap_top_index(); // lien précédent
// Initialiser cache
{
opt->state._hts_in_html_parsing = 4;
if (!RUN_CALLBACK7(opt, loop, NULL, 0, 0, 0, opt->lien_tot, 0, NULL)) {
opt->state.exit_xh = 1; // exit requested
}
cache_init(&cache, opt);
opt->state._hts_in_html_parsing = 0;
}
}
#if BDEBUG==3
{
int i;
for(i = 0; i < lien_tot; i++) {
printf("%d>%s%s as %s\n", i, heap(i)->adr, heap(i)->fil, heap(i)->sav);
}
for(i = 0; i < filptr; i++) {
printf("%d>filters=%s\n", i, filters[i]);
}
}
#endif
// backing
//soc_max=opt->maxsoc;
if (opt->maxsoc > 0) {
#if BDEBUG==2
_CLRSCR;
#endif
// Nombre de fichiers HTML pouvant être présents en mémoire de manière simultannée
// On prévoit large: les fichiers HTML ne prennent que peu de place en mémoire, et les
// fichiers non html sont sauvés en direct sur disque.
// --> 1024 entrées + 32 entrées par socket en supplément
sback = back_new(opt, opt->maxsoc * 32 + 1024);
if (sback == NULL) {
hts_log_print(opt, LOG_PANIC,
"Not enough memory, can not allocate %d bytes",
(int) ((opt->maxsoc + 1) * sizeof(lien_back)));
return 0;
}
}
// statistiques
if (opt->makestat) {
makestat_fp =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-stats.txt"),
"wb");
if (makestat_fp != NULL) {
fprintf(makestat_fp, "HTTrack statistics report, every minutes" LF LF);
fflush(makestat_fp);
}
}
// tracking -- débuggage
if (opt->maketrack) {
maketrack_fp =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-track.txt"),
"wb");
if (maketrack_fp != NULL) {
fprintf(maketrack_fp, "HTTrack tracking report, every minutes" LF LF);
fflush(maketrack_fp);
}
}
// on n'a pas de liens!! (exemple: httrack www.* impossible sans départ..)
if (opt->lien_tot <= 0) {
hts_log_print(opt, LOG_ERROR,
"You MUST specify at least one complete URL, and not only wildcards!");
}
/* Send options to callback functions */
RUN_CALLBACK0(opt, chopt);
// attendre une certaine heure..
if (opt->waittime > 0) {
int rollover = 0;
int ok = 0;
{
TStamp tl = 0;
time_t tt;
struct tm *A;
tt = time(NULL);
A = localtime(&tt);
tl += A->tm_sec;
tl += A->tm_min * 60;
tl += A->tm_hour * 60 * 60;
if (tl > opt->waittime) // attendre minuit
rollover = 1;
}
// attendre..
opt->state._hts_in_html_parsing = 5;
do {
TStamp tl = 0;
time_t tt;
struct tm *A;
tt = time(NULL);
A = localtime(&tt);
tl += A->tm_sec;
tl += A->tm_min * 60;
tl += A->tm_hour * 60 * 60;
if (rollover) {
if (tl <= opt->waittime)
rollover = 0; // attendre heure
} else {
if (tl > opt->waittime)
ok = 1; // ok!
}
{
int r;
if (rollover)
r =
RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, 0, 0, opt->lien_tot,
(int) (opt->waittime - tl + 24 * 3600), NULL);
else
r =
RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, 0, 0, opt->lien_tot,
(int) (opt->waittime - tl), NULL);
if (!r) {
opt->state.exit_xh = 1; // exit requested
ok = 1;
} else
Sleep(100);
}
} while(!ok);
opt->state._hts_in_html_parsing = 0;
// note: recopie de plus haut
// noter heure actuelle de départ en secondes
HTS_STAT.stat_timestart = time_local();
if (opt->shell) {
last_info_shell = HTS_STAT.stat_timestart;
}
if ((opt->makestat) || (opt->maketrack)) {
makestat_time = HTS_STAT.stat_timestart;
}
}
/* Info for wrappers */
hts_log_print(opt, LOG_INFO, "engine: start");
if (!RUN_CALLBACK0(opt, start)) {
XH_extuninit;
return 1;
}
// ------------------------------------------------------------
// ------------------------------------------------------------
// Boucle générale de parcours des liens
// ------------------------------------------------------------
do {
int error = 0; // si error alors sauter
int store_errpage = 0; // c'est une erreur mais on enregistre le html
int is_binary = 0; // is a binary file
int is_loaded_from_file = 0; // has been loaded from a file (implies is_write=1)
char BIGSTK loc[HTS_URLMAXSIZE * 2]; // adresse de relocation
// Ici on charge le fichier (html, gif..) en mémoire
// Les HTMLs sont traités (si leur priorité est suffisante)
// effacer r
hts_init_htsblk(&r);
//memset(&r, 0, sizeof(htsblk)); r.soc=INVALID_SOCKET;
r.location = loc; // en cas d'erreur 3xx (moved)
// recopier proxy
if ((r.req.proxy.active = opt->proxy.active)) {
if (StringBuff(opt->proxy.bindhost) != NULL)
r.req.proxy.bindhost = StringBuff(opt->proxy.bindhost);
if (StringBuff(opt->proxy.name) != NULL)
r.req.proxy.name = StringBuff(opt->proxy.name);
r.req.proxy.port = opt->proxy.port;
}
// et user-agent
r.req.user_agent = StringBuff(opt->user_agent);
r.req.referer = StringBuff(opt->referer);
r.req.from = StringBuff(opt->from);
r.req.lang_iso = StringBuff(opt->lang_iso);
r.req.accept = StringBuff(opt->accept);
r.req.headers = StringBuff(opt->headers);
r.req.user_agent_send = opt->user_agent_send;
if (!error) {
// Skip empty/invalid/done in background
if (heap(ptr)) {
while(ptr < opt->lien_tot
&& (heap(ptr))
&& ((((urladr() != NULL) ? (urladr()) : (" "))[0] == '!')
|| (((urlfil() != NULL) ? (urlfil()) : (" "))[0] == '\0')
|| ((heap(ptr)->pass2 == -1))
)
) { // sauter si lien annulé (ou fil vide)
if (heap(ptr) != NULL && heap(ptr)->pass2 == -1) {
hts_log_print(opt, LOG_DEBUG, "link #%d is ready, skipping: %s%s..",
ptr, ((urladr() != NULL) ? (urladr()) : (" ")),
((urlfil() != NULL) ? (urlfil()) : (" ")));
} else {
hts_log_print(opt, LOG_DEBUG,
"link #%d seems ready, skipping: %s%s..", ptr,
((urladr() != NULL) ? (urladr()) : (" ")),
((urlfil() != NULL) ? (urlfil()) : (" ")));
}
// remove from stats
if (heap(ptr)->pass2 == -1) {
HTS_STAT.stat_background--;
}
ptr++;
}
// We're done!
if (ptr == opt->lien_tot) {
goto jump_if_done;
}
}
if (heap(ptr) != NULL) { // on a qq chose à récupérer?
hts_log_print(opt, LOG_DEBUG, "Wait get: %s%s", urladr(), urlfil());
#if DEBUG_ROBOTS
if (strcmp(urlfil(), "/robots.txt") == 0) {
printf("robots.txt detected\n");
}
#endif
// ------------------------------------------------------------
// DEBUT --RECUPERATION LIEN---
if (ptr == 0) { // premier lien à parcourir: lien primaire construit avant
r.adr = primary;
primary = NULL;
r.statuscode = HTTP_OK;
r.size = strlen(r.adr);
r.soc = INVALID_SOCKET;
strcpybuff(r.contenttype, "text/html");
/*} else if (opt->maxsoc<=0) { // fichiers 1 à 1 en attente (pas de backing)
// charger le fichier en mémoire tout bêtement
r=xhttpget(urladr(),urlfil());
//
*/
} else { // backing, multiples sockets
/*
**************************************
Get the next link, waiting for other files, handling external callbacks
*/
{
char BIGSTK buff_err_msg[1024];
htsmoduleStruct BIGSTK str;
htsmoduleStructExtended BIGSTK stre;
buff_err_msg[0] = '\0';
memset(&str, 0, sizeof(str));
memset(&stre, 0, sizeof(stre));
/* */
str.err_msg = buff_err_msg;
str.filename = savename();
str.mime = r.contenttype;
str.url_host = urladr();
str.url_file = urlfil();
str.size = (const int) r.size;
/* */
str.addLink = htsAddLink;
/* */
str.opt = opt;
str.sback = sback;
str.cache = &cache;
str.hashptr = hashptr;
str.numero_passe = numero_passe;
/* */
str.ptr_ = &ptr;
/* */
str.page_charset_ = NULL;
/* */
/* */
stre.r_ = &r;
/* */
stre.error_ = &error;
stre.exit_xh_ = &opt->state.exit_xh;
stre.store_errpage_ = &store_errpage;
/* */
stre.base = base;
stre.codebase = codebase;
/* */
stre.filters_ = &filters;
stre.filptr_ = &filptr;
stre.robots_ = &robots;
stre.hash_ = &hash;
/* */
stre.makeindex_done_ = &makeindex_done;
stre.makeindex_fp_ = &makeindex_fp;
stre.makeindex_links_ = &makeindex_links;
stre.makeindex_firstlink_ = makeindex_firstlink;
/* */
stre.template_header_ = template_header;
stre.template_body_ = template_body;
stre.template_footer_ = template_footer;
/* */
stre.stat_fragment_ = &stat_fragment;
stre.makestat_time = makestat_time;
stre.makestat_fp = makestat_fp;
stre.makestat_total_ = &makestat_total;
stre.makestat_lnk_ = &makestat_lnk;
stre.maketrack_fp = maketrack_fp;
/* FUNCTION DEPENDANT */
stre.loc_ = loc;
stre.last_info_shell_ = &last_info_shell;
stre.info_shell_ = &info_shell;
/* Parse */
switch (hts_mirror_wait_for_next_file(&str, &stre)) {
case -1:
XH_uninit;
return -1;
break;
case 2:
// Jump to 'continue'
// This is one of the very very rare cases where goto
// is acceptable
// A supplemental flag and if( ) { } would be really messy
goto jump_if_done;
}
}
}
// FIN --RECUPERATION LIEN---
// ------------------------------------------------------------
} else { // lien vide..
hts_log_print(opt, LOG_WARNING, "Warning, link #%d empty", ptr);
error = 1;
goto jump_if_done;
} // test si url existe (non vide!)
// ---tester taille a posteriori---
// tester r.adr
if (!error) {
// erreur, pas de fichier chargé:
if ((!r.adr) && (r.is_write == 0)
&& (r.statuscode != 301)
&& (r.statuscode != 302)
&& (r.statuscode != 303)
&& (r.statuscode != 307)
&& (r.statuscode != 412)
&& (r.statuscode != 416)
) {
// error=1;
// peut être que le fichier était trop gros?
if ((istoobig
(opt, r.totalsize, opt->maxfile_html, opt->maxfile_nonhtml,
r.contenttype))
||
(istoobig
(opt, r.totalsize, opt->maxfile_html, opt->maxfile_nonhtml,
r.contenttype))) {
error = 0;
hts_log_print(opt, LOG_WARNING,
"Big file cancelled according to user's preferences: %s%s",
urladr(), urlfil());
}
// // // error=1; // ne pas traiter la suite -- euhh si finalement..
}
}
// ---fin tester taille a posteriori---
// --------------------
// BOGUS MIME TYPE HACK
// Check if we have a bogus MIME type
// example:
// Content-type="text/html"
// and
// Content-disposition="foo.jpg"
// --------------------
if (!error) {
if (r.statuscode == HTTP_OK) { // OK (ou 304 en backing)
if (r.adr) { // Written file
// Buggy SVG (Smiling Spectre)
if (strcmp(r.contenttype, "image/svg+xml") == 0 && !look_like_xml(r.adr)) {
// patch it
strcpybuff(r.contenttype, "application/octet-stream");
is_binary = 1;
}
else if ((is_hypertext_mime(opt, r.contenttype, urlfil()))
/* Is HTML or Js, .. */
/* NO - real media is real media, not HTML */
/*|| (may_be_hypertext_mime(r.contenttype, urlfil()) && (r.adr) ) */
/* Is real media, .. */
) {
if (strnotempty(r.cdispo)) { // Content-disposition set!
if (ishtml(opt, savename()) == 0) { // Non HTML!!
// patch it!
strcpybuff(r.contenttype, "application/octet-stream");
}
}
}
}
}
/* Load file if necessary and decode. */
#define LOAD_IN_MEMORY_IF_NECESSARY() do { \
if ( \
may_be_hypertext_mime(opt,r.contenttype, urlfil()) /* Is HTML or Js, .. */ \
&& (heap(ptr)->depth>0) /* Depth > 0 (recurse depth) */ \
&& (r.adr==NULL) /* HTML Data exists */ \
&& (!store_errpage) /* Not an html error page */ \
&& (savename()[0]!='\0') /* Output filename exists */ \
) \
{ \
is_loaded_from_file = 1; \
r.adr = readfile2(savename(), &r.size); \
if (r.adr != NULL) { \
hts_log_print(opt, LOG_INFO, "File successfully loaded for parsing: %s%s (%d bytes)",urladr(),urlfil(),(int)r.size); \
} else { \
hts_log_print(opt, LOG_ERROR, "File could not be loaded for parsing: %s%s",urladr(),urlfil()); \
} \
} \
} while(0)
/* Load file and decode if necessary, before content-binary check. (3.43) */
LOAD_IN_MEMORY_IF_NECESSARY();
// ------------------------------------
// BOGUS MIME TYPE HACK II (the revenge)
// Check if we have a bogus MIME type
if (HTTP_IS_OK(r.statuscode) && (is_hypertext_mime(opt, r.contenttype, urlfil()) /* Is HTML or Js, .. */
||may_be_hypertext_mime(opt, r.contenttype, urlfil())) /* Is real media, .. */
) {
/* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
//if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
// char *charset;
// char *pos;
// if (r.charset[0] != '\0') {
// charset = strdup(r.charset);
// } else {
// charset = hts_getCharsetFromMeta(r.adr, r.size);
// }
// if (charset != NULL) {
// char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
// /* Use new buffer */
// if (utf8 != NULL) {
// freet(r.adr);
// r.size = strlen(utf8);
// r.adr = utf8;
// /* New UTF-8 charset */
// r.charset[0] = '\0';
// strcpy(r.charset, "utf-8");
// }
// /* Free charset */
// free(charset);
// }
//}
/* Check bogus chars */
if ((r.adr) && (r.size)) {
unsigned int map[256];
int i;
unsigned int nspec = 0;
map_characters((unsigned char *) r.adr, (unsigned int) r.size,
&map[0]);
for(i = 1; i < 32; i++) { // null chars ignored..
if (!is_realspace(i)
&& i != 27 /* Damn you ISO2022-xx! */
) {
nspec += map[i];
}
}
/* On-the-fly UCS2 to UTF-8 conversion (note: UCS2 should never be used on the net) */
if (map[0] > r.size / 10 && r.size % 2 == 0
&&
((((unsigned char) r.adr[0]) == 0xff
&& ((unsigned char) r.adr[1]) == 0xfe)
|| (((unsigned char) r.adr[0]) == 0xfe
&& ((unsigned char) r.adr[1]) == 0xff)
)
) {
#define CH_ADD(c) do { \
if (new_offs + 1 > new_capa) { \
new_capa *= 2; \
new_adr = (unsigned char*) realloct(new_adr, \
new_capa); \
assertf(new_adr != NULL); \
} \
new_adr[new_offs++] = (unsigned char) (c); \
} while(0)
#define CH_ADD_RNG1(c, r, o) do { \
CH_ADD( (c) / (r) + (o) ); \
c = (c) % (r); \
} while(0)
#define CH_ADD_RNG0(c, o) do { \
CH_ADD_RNG1(c, 1, o); \
} while(0)
#define CH_ADD_RNG2(c, r, r2, o) do { \
CH_ADD_RNG1(c, (r) * (r2), o); \
} while(0)
int new_capa = (int) (r.size / 2 + 1);
int new_offs = 0;
unsigned char *prev_adr = (unsigned char *) r.adr;
unsigned char *new_adr = (unsigned char *) malloct(new_capa);
int i;
int swap = (((unsigned char) r.adr[0]) == 0xff);
assertf(new_adr != NULL);
/*
See http://www.unicode.org/reports/tr28/tr28-3.html#conformance
U+0000..U+007F 00..7F
U+0080..U+07FF C2..DF 80..BF
U+0800..U+0FFF E0 A0..BF 80..BF
U+1000..U+CFFF E1..EC 80..BF 80..BF
U+D000..U+D7FF ED 80..9F 80..BF
U+D800..U+DFFF
U+E000..U+FFFF EE..EF 80..BF 80..BF
*/
for(i = 0; i < r.size / 2; i++) {
unsigned short int unic = 0;
if (swap)
unic = prev_adr[i * 2] + (prev_adr[i * 2 + 1] << 8);
else
unic = (prev_adr[i * 2] << 8) + prev_adr[i * 2 + 1];
if (unic <= 0x7F) {
/* U+0000..U+007F 00..7F */
CH_ADD_RNG0(unic, 0x00);
} else if (unic <= 0x07FF) {
/* U+0080..U+07FF C2..DF 80..BF */
unic -= 0x0080;
CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0xc2);
CH_ADD_RNG0(unic, 0x80);
} else if (unic <= 0x0FFF) {
/* U+0800..U+0FFF E0 A0..BF 80..BF */
unic -= 0x0800;
CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0xbf - 0xa0 + 1, 0xe0);
CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0xa0);
CH_ADD_RNG0(unic, 0x80);
} else if (unic <= 0xCFFF) {
/* U+1000..U+CFFF E1..EC 80..BF 80..BF */
unic -= 0x1000;
CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xe1);
CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0x80);
CH_ADD_RNG0(unic, 0x80);
} else if (unic <= 0xD7FF) {
/* U+D000..U+D7FF ED 80..9F 80..BF */
unic -= 0xD000;
CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0x9f - 0x80 + 1, 0xed);
CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0x80);
CH_ADD_RNG0(unic, 0x80);
} else if (unic <= 0xDFFF) {
/* U+D800..U+DFFF */
CH_ADD('?');
/* ill-formed */
} else { /* if (unic <= 0xFFFF) */
/* U+E000..U+FFFF EE..EF 80..BF 80..BF */
unic -= 0xE000;
CH_ADD_RNG2(unic, 0xbf - 0x80 + 1, 0xbf - 0x80 + 1, 0xee);
CH_ADD_RNG1(unic, 0xbf - 0x80 + 1, 0x80);
CH_ADD_RNG0(unic, 0x80);
}
}
hts_log_print(opt, LOG_WARNING,
"File %s%s converted from UCS2 to UTF-8 (old size: %d bytes, new size: %d bytes)",
urladr(), urlfil(), (int) r.size, new_offs);
freet(r.adr);
r.adr = NULL;
r.size = new_offs;
CH_ADD(0);
r.adr = (char *) new_adr;
#undef CH_ADD
#undef CH_ADD_RNG0
#undef CH_ADD_RNG1
#undef CH_ADD_RNG2
} else if ((nspec > r.size / 100) && (nspec > 10)) { // too many special characters
is_binary = 1;
strcpybuff(r.contenttype, "application/octet-stream");
hts_log_print(opt, LOG_WARNING,
"File not parsed, looks like binary: %s%s", urladr(),
urlfil());
}
/* This hack allows you to avoid problems with parsing '\0' characters */
if (!is_binary) {
for(i = 0; i < r.size; i++) {
if (r.adr[i] == '\0')
r.adr[i] = ' ';
}
}
}
}
}
// MOVED IN back_finalize()
//
// --------------------
// REAL MEDIA HACK
// Check if we have to load locally the file
// --------------------
//if (!error) {
// if (r.statuscode == HTTP_OK) { // OK (ou 304 en backing)
// if (r.adr==NULL) { // Written file
// if (may_be_hypertext_mime(r.contenttype, urlfil())) { // to parse!
// LLint sz;
// sz=fsize_utf8(savename());
// if (sz>0) { // ok, exists!
// if (sz < 8192) { // ok, small file --> to parse!
// FILE* fp=FOPEN(savename(),"rb");
// if (fp) {
// r.adr=malloct(sz + 1);
// if (r.adr) {
// if (fread(r.adr,1,sz,fp) == sz) {
// r.size=sz;
// r.adr[sz] = '\0';
// r.is_write = 0;
// } else {
// freet(r.adr);
// r.size=0;
// r.adr = NULL;
// r.statuscode=STATUSCODE_INVALID;
// strcpybuff(r.msg, ".RAM read error");
// }
// fclose(fp);
// fp=NULL;
// // remove (temporary) file!
// remove(savename());
// }
// if (fp)
// fclose(fp);
// }
// }
// }
// }
// }
// }
//}
// EN OF REAL MEDIA HACK
// ---stockage en cache---
// stocker dans le cache?
/*
if (!error) {
if (ptr>0) {
if (heap(ptr)) {
xxcache_mayadd(opt,&cache,&r,urladr(),urlfil(),savename());
} else
error=1;
}
}
*/
// ---fin stockage en cache---
/*
**************************************
Check "Moved permanently" and other similar errors, retrying URLs if necessary and handling
redirect pages.
*/
if (!error) {
char BIGSTK buff_err_msg[1024];
htsmoduleStruct BIGSTK str;
htsmoduleStructExtended BIGSTK stre;
buff_err_msg[0] = '\0';
memset(&str, 0, sizeof(str));
memset(&stre, 0, sizeof(stre));
/* */
str.err_msg = buff_err_msg;
str.filename = savename();
str.mime = r.contenttype;
str.url_host = urladr();
str.url_file = urlfil();
str.size = (int) r.size;
/* */
str.addLink = htsAddLink;
/* */
str.opt = opt;
str.sback = sback;
str.cache = &cache;
str.hashptr = hashptr;
str.numero_passe = numero_passe;
/* */
str.ptr_ = &ptr;
/* */
str.page_charset_ = NULL;
/* */
/* */
stre.r_ = &r;
/* */
stre.error_ = &error;
stre.exit_xh_ = &opt->state.exit_xh;
stre.store_errpage_ = &store_errpage;
/* */
stre.base = base;
stre.codebase = codebase;
/* */
stre.filters_ = &filters;
stre.filptr_ = &filptr;
stre.robots_ = &robots;
stre.hash_ = &hash;
/* */
stre.makeindex_done_ = &makeindex_done;
stre.makeindex_fp_ = &makeindex_fp;
stre.makeindex_links_ = &makeindex_links;
stre.makeindex_firstlink_ = makeindex_firstlink;
/* */
stre.template_header_ = template_header;
stre.template_body_ = template_body;
stre.template_footer_ = template_footer;
/* */
stre.stat_fragment_ = &stat_fragment;
stre.makestat_time = makestat_time;
stre.makestat_fp = makestat_fp;
stre.makestat_total_ = &makestat_total;
stre.makestat_lnk_ = &makestat_lnk;
stre.maketrack_fp = maketrack_fp;
/* Parse */
if (hts_mirror_check_moved(&str, &stre) != 0) {
XH_uninit;
return -1;
}
}
} // if !error
if (!error) {
#if DEBUG_SHOWTYPES
if (strstr(REG, r.contenttype) == NULL) {
strcatbuff(REG, r.contenttype);
strcatbuff(REG, "\n");
printf("%s\n", r.contenttype);
io_flush;
}
#endif
/* Load file and decode if necessary, after redirect check. */
LOAD_IN_MEMORY_IF_NECESSARY();
// ------------------------------------------------------
// ok, fichier chargé localement
// ------------------------------------------------------
// Vérificateur d'intégrité
#if DEBUG_CHECKINT
{
int i;
for(i = 0; i < sback->count; i++) {
char si[256];
sprintf(si, "Test global après back_wait, index %d", i);
_CHECKINT(&back[i], si)
}
}
#endif
/* info: updated */
/*
if (ptr>0) {
// "mis à jour"
if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) { // page modifiée
if (strnotempty(savename())) {
HTS_STAT.stat_updated_files++;
//if ((opt->debug>0) && (opt->log!=NULL)) {
hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
}
} else {
if (!store_errpage) {
hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
}
}
}
*/
// ------------------------------------------------------
// traitement (parsing)
// ------------------------------------------------------
// traiter
if (!is_binary && ((is_hypertext_mime(opt, r.contenttype, urlfil())) /* Is HTML or Js, .. */
||(may_be_hypertext_mime(opt, r.contenttype, urlfil()) && r.adr != NULL) /* Is real media, .. */
)
&& (heap(ptr)->depth > 0) /* Depth > 0 (recurse depth) */
&&(r.adr != NULL) /* HTML Data exists */
&&(r.size > 0) /* And not empty */
&&(!store_errpage) /* Not an html error page */
&&(savename()[0] != '\0') /* Output filename exists */
) { // ne traiter que le html si autorisé
// -- -- -- --
// Parsing HTML
if (!error) {
char page_charset[32];
/* Remove file if being processed */
if (is_loaded_from_file) {
(void) unlink(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), savename()));
is_loaded_from_file = 0;
}
/* Detect charset to convert links into proper UTF8 filenames */
page_charset[0] = '\0';
if (opt->convert_utf8) {
/* HTTP charset is prioritary over meta */
if (r.charset[0] != '\0') {
if (strlen(r.charset) < sizeof(page_charset)) {
strcpy(page_charset, r.charset);
}
}
/* Attempt to find a meta charset */
else if (is_html_mime_type(r.contenttype)) {
char *const charset = hts_getCharsetFromMeta(r.adr, r.size);
if (charset != NULL && strlen(charset) < sizeof(page_charset)) {
strcpy(page_charset, charset);
}
if (charset != NULL)
free(charset);
}
/* Could not detect charset: could it be UTF-8 ? */
/* No, we can not do that: browsers do not do it
(and it would break links). */
//if (page_charset[0] == '\0') {
// if (is_unicode_utf8(r.adr, r.size)) {
// strcpy(page_charset, "utf-8");
// }
//}
/* Could not detect charset */
if (page_charset[0] == '\0') {
hts_log_print(opt, LOG_INFO,
"Warning: could not detect encoding for: %s%s",
urladr(), urlfil());
/* Fallback to ISO-8859-1 (~== identity) ; accents will look weird */
strcpy(page_charset, "iso-8859-1");
}
}
/* Info for wrappers */
hts_log_print(opt, LOG_INFO, "engine: check-html: %s%s", urladr(),
urlfil());
{
char BIGSTK buff_err_msg[1024];
htsmoduleStruct BIGSTK str;
htsmoduleStructExtended BIGSTK stre;
buff_err_msg[0] = '\0';
memset(&str, 0, sizeof(str));
memset(&stre, 0, sizeof(stre));
/* */
str.err_msg = buff_err_msg;
str.filename = savename();
str.mime = r.contenttype;
str.url_host = urladr();
str.url_file = urlfil();
str.size = (int) r.size;
/* */
str.addLink = htsAddLink;
/* */
str.opt = opt;
str.sback = sback;
str.cache = &cache;
str.hashptr = hashptr;
str.numero_passe = numero_passe;
/* */
str.ptr_ = &ptr;
/* */
str.page_charset_ = page_charset[0] != '\0' ? page_charset : NULL;
/* */
/* */
stre.r_ = &r;
/* */
stre.error_ = &error;
stre.exit_xh_ = &opt->state.exit_xh;
stre.store_errpage_ = &store_errpage;
/* */
stre.base = base;
stre.codebase = codebase;
/* */
stre.filters_ = &filters;
stre.filptr_ = &filptr;
stre.robots_ = &robots;
stre.hash_ = &hash;
/* */
stre.makeindex_done_ = &makeindex_done;
stre.makeindex_fp_ = &makeindex_fp;
stre.makeindex_links_ = &makeindex_links;
stre.makeindex_firstlink_ = makeindex_firstlink;
/* */
stre.template_header_ = template_header;
stre.template_body_ = template_body;
stre.template_footer_ = template_footer;
/* */
stre.stat_fragment_ = &stat_fragment;
stre.makestat_time = makestat_time;
stre.makestat_fp = makestat_fp;
stre.makestat_total_ = &makestat_total;
stre.makestat_lnk_ = &makestat_lnk;
stre.maketrack_fp = maketrack_fp;
/* Parse */
if (htsparse(&str, &stre) != 0) {
XH_uninit;
return -1;
}
// I'll have to segment this part
// #include "htsparse.c"
}
}
// Fin parsing HTML
// -- -- -- --
} // si text/html
// -- -- --
else { // sauver fichier quelconque
// -- -- --
// sauver fichier
/* En cas d'erreur, vérifier que fichier d'erreur existe */
if (strnotempty(savename()) == 0) { // chemin de sauvegarde existant
if (strcmp(urlfil(), "/robots.txt") == 0) { // pas robots.txt
if (store_errpage) { // c'est une page d'erreur
int create_html_warning = 0;
int create_gif_warning = 0;
switch (ishtml(opt, urlfil())) { /* pas fichier html */
case 0: /* non html */
{
char buff[256];
guess_httptype(opt, buff, urlfil());
if (strcmp(buff, "image/gif") == 0)
create_gif_warning = 1;
}
break;
case 1: /* html */
if (!r.adr) {
}
break;
default: /* don't know.. */
break;
}
/* Créer message d'erreur ? */
if (create_html_warning) {
char *adr =
(char *) malloct(strlen(HTS_DATA_ERROR_HTML) + 1100);
hts_log_print(opt, LOG_INFO, "Creating HTML warning file (%s)",
r.msg);
if (adr) {
if (r.adr) {
freet(r.adr);
r.adr = NULL;
}
sprintf(adr, HTS_DATA_ERROR_HTML, r.msg);
r.adr = adr;
}
} else if (create_gif_warning) {
char *adr = (char *) malloct(HTS_DATA_UNKNOWN_GIF_LEN);
hts_log_print(opt, LOG_INFO, "Creating GIF dummy file (%s)",
r.msg);
if (r.adr) {
freet(r.adr);
r.adr = NULL;
}
memcpy(adr, HTS_DATA_UNKNOWN_GIF, HTS_DATA_UNKNOWN_GIF_LEN);
r.adr = adr;
}
}
}
}
if (strnotempty(savename()) == 0) { // pas de chemin de sauvegarde
if (strcmp(urlfil(), "/robots.txt") == 0) { // robots.txt
if (r.adr) {
int bptr = 0;
char BIGSTK line[1024];
char BIGSTK buff[8192];
char BIGSTK infobuff[8192];
int record = 0;
line[0] = '\0';
buff[0] = '\0';
infobuff[0] = '\0';
//
#if DEBUG_ROBOTS
printf("robots.txt dump:\n%s\n", r.adr);
#endif
do {
char *comm;
int llen;
bptr += binput(r.adr + bptr, line, sizeof(line) - 2);
/* strip comment */
comm = strchr(line, '#');
if (comm != NULL) {
*comm = '\0';
}
/* strip spaces */
llen = (int) strlen(line);
while(llen > 0 && is_realspace(line[llen - 1])) {
line[llen - 1] = '\0';
llen--;
}
if (strfield(line, "user-agent:")) {
char *a;
a = line + 11;
while(is_realspace(*a))
a++; // sauter espace(s)
if (*a == '*') {
if (record != 2)
record = 1; // c pour nous
} else if (strfield(a, "httrack") || strfield(a, "winhttrack")
|| strfield(a, "webhttrack")) {
buff[0] = '\0'; // re-enregistrer
infobuff[0] = '\0';
record = 2; // locked
#if DEBUG_ROBOTS
printf("explicit disallow for httrack\n");
#endif
} else
record = 0;
} else if (record) {
if (strfield(line, "disallow:")) {
char *a = line + 9;
while(is_realspace(*a))
a++; // sauter espace(s)
if (strnotempty(a)) {
#ifdef IGNORE_RESTRICTIVE_ROBOTS
if (strcmp(a, "/") != 0 || opt->robots >= 3)
#endif
{ /* ignoring disallow: / */
if ((strlen(buff) + strlen(a) + 8) < sizeof(buff)) {
strcatbuff(buff, a);
strcatbuff(buff, "\n");
if ((strlen(infobuff) + strlen(a) + 8) <
sizeof(infobuff)) {
if (strnotempty(infobuff))
strcatbuff(infobuff, ", ");
strcatbuff(infobuff, a);
}
}
}
#ifdef IGNORE_RESTRICTIVE_ROBOTS
else {
hts_log_print(opt, LOG_NOTICE,
"Note: %s robots.txt rules are too restrictive, ignoring /",
urladr());
}
#endif
}
}
}
} while((bptr < r.size) && (strlen(buff) < (sizeof(buff) - 32)));
if (strnotempty(buff)) {
checkrobots_set(&robots, urladr(), buff);
hts_log_print(opt, LOG_INFO,
"Note: robots.txt forbidden links for %s are: %s",
urladr(), infobuff);
hts_log_print(opt, LOG_NOTICE,
"Note: due to %s remote robots.txt rules, links beginning with these path will be forbidden: %s (see in the options to disable this)",
urladr(), infobuff);
}
}
}
} else if (r.is_write) { // déja sauvé sur disque
/*
if (!ishttperror(r.statuscode))
HTS_STAT.stat_files++;
HTS_STAT.stat_bytes+=r.size;
*/
//printf("ok......\n");
} else {
// Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
// récursion nous en empêche
// Dans ce cas on met un fichier indiquant ce fait
// Si par la suite on doit retraiter ce fichier avec un niveau de récursion plus
// fort, on supprimera le readme, et on scannera le fichier html!
// note: sauté si store_errpage (càd si page d'erreur, non à scanner!)
if ((is_hypertext_mime(opt, r.contenttype, urlfil())) && (!store_errpage) && (r.size > 0)) { // c'est du html!!
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
FILE *fp;
tempo[0] = '\0';
strcpybuff(tempo, savename());
strcatbuff(tempo, ".readme");
#if HTS_DOSNAME
// remplacer / par des slash arrière
{
int i = 0;
while(tempo[i]) {
if (tempo[i] == '/')
tempo[i] = '\\';
i++;
}
}
// a partir d'ici le slash devient antislash
#endif
if ((fp = FOPEN(tempo, "wb")) != NULL) {
fprintf(fp,
"Info-file generated by HTTrack Website Copier "
HTTRACK_VERSION "%s" CRLF "" CRLF,
hts_get_version_info(opt));
fprintf(fp, "The file %s has not been scanned by HTS" CRLF,
savename());
fprintf(fp,
"Some links contained in it may be unreachable locally."
CRLF);
fprintf(fp,
"If you want to get these files, you have to set an upper recurse level, ");
fprintf(fp, "and to rescan the URL." CRLF);
fclose(fp);
#ifndef _WIN32
chmod(tempo, HTS_ACCESS_FILE);
#endif
usercommand(opt, 0, NULL, fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), tempo), "",
"");
}
hts_log_print(opt, LOG_WARNING,
"Warning: store %s without scan: %s", r.contenttype,
savename());
} else {
if ((opt->getmode & 2) != 0) { // ok autorisé
hts_log_print(opt, LOG_DEBUG, "Store %s: %s", r.contenttype,
savename());
} else { // lien non autorisé! (ex: cgi-bin en html)
hts_log_print(opt, LOG_DEBUG,
"non-html file ignored after upload at %s : %s",
urladr(), urlfil());
if (r.adr) {
freet(r.adr);
r.adr = NULL;
}
}
}
//printf("extern=%s\n",r.contenttype);
// ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
if (r.adr != NULL || r.size == 0) {
file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
0) {
int fcheck;
if ((fcheck = check_fatal_io_errno())) {
hts_log_print(opt, LOG_ERROR,
"Mirror aborted: disk full or filesystem problems");
opt->state.exit_xh = -1; /* fatal error */
}
hts_log_print(opt, LOG_ERROR | LOG_ERRNO,
"Unable to save file %s", savename());
} else {
/*
if (!ishttperror(r.statuscode))
HTS_STAT.stat_files++;
HTS_STAT.stat_bytes+=r.size;
*/
}
}
}
/* Parsing of other media types (java, ram..) */
/*
if (strfield2(r.contenttype,"audio/x-pn-realaudio")) {
hts_log_print(opt, LOG_DEBUG, "(Real Media): parsing %s",savename());
if (fexist(savename())) { // ok, existe bien!
FILE* fp=FOPEN(savename(),"r+b");
if (fp) {
if (!fseek(fp,0,SEEK_SET)) {
char BIGSTK line[HTS_URLMAXSIZE*2];
linput(fp,line,HTS_URLMAXSIZE);
if (strnotempty(line)) {
hts_log_print(opt, LOG_DEBUG, "(Real Media): detected %s",line);
}
}
fclose(fp);
}
}
} else */
/* External modules */
if (opt->parsejava && (opt->parsejava & HTSPARSE_NO_CLASS) == 0
&& fexist(savename())) {
char BIGSTK buff_err_msg[1024];
htsmoduleStruct BIGSTK str;
buff_err_msg[0] = '\0';
memset(&str, 0, sizeof(str));
/* */
str.err_msg = buff_err_msg;
str.filename = savename();
str.mime = r.contenttype;
str.url_host = urladr();
str.url_file = urlfil();
str.size = (int) r.size;
/* */
str.addLink = htsAddLink;
/* */
str.opt = opt;
str.sback = sback;
str.cache = &cache;
str.hashptr = hashptr;
str.numero_passe = numero_passe;
str.ptr_ = &ptr;
/* Parse if recognized */
switch (hts_parse_externals(&str)) {
case 1:
hts_log_print(opt, LOG_DEBUG,
"(External module): parsed successfully %s",
savename());
break;
case 0:
hts_log_print(opt, LOG_DEBUG,
"(External module): couldn't parse successfully %s : %s",
savename(), str.err_msg);
break;
}
}
} // text/html ou autre
/* Post-processing */
if (fexist(savename())) {
usercommand(opt, 0, NULL, savename(), urladr(), urlfil());
}
} // if !error
jump_if_done:
// libérer les liens
if (r.adr) {
freet(r.adr);
r.adr = NULL;
} // libérer la mémoire!
// prochain lien
ptr++;
// faut-il sauter le(s) lien(s) suivant(s)? (fichiers images à passer après les html)
if (opt->getmode & 4) { // sauver les non html après
// sauter les fichiers selon la passe
if (!numero_passe) {
while((ptr < opt->lien_tot) ? (heap(ptr)->pass2) : 0)
ptr++;
} else {
while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
ptr++;
}
if (ptr >= opt->lien_tot) { // fin de boucle
if (!numero_passe) { // première boucle
hts_log_print(opt, LOG_DEBUG, "Now getting non-html files...");
numero_passe = 1; // seconde boucle
ptr = 0;
// prochain pass2
while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
ptr++;
//printf("first link==%d\n");
}
}
}
// copy abort state if necessary from outside
//if (!exit_xh && opt->state.exit_xh) {
// exit_xh=opt->state.exit_xh;
//}
// a-t-on dépassé le quota?
if (!back_checkmirror(opt)) {
ptr = opt->lien_tot;
} else if (opt->state.exit_xh) { // sortir
if (opt->state.exit_xh == 1) {
hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
} else {
hts_log_print(opt, LOG_ERROR, "Exit requested by engine");
}
ptr = opt->lien_tot;
}
} while(ptr < opt->lien_tot);
//
//
//
/*
Ensure the index is being closed
*/
HT_INDEX_END;
/*
updating-a-remotely-deteted-website hack
no much data transferred, no data saved
<no files successfulyl saved>
we assume that something was bad (no connection)
just backup old cache and restore everything
*/
if ((HTS_STAT.stat_files <= 0)
&& (HTS_STAT.HTS_TOTAL_RECV < 32768) /* should be fine */
) {
hts_log_print(opt, LOG_NOTICE,
"No data seems to have been transferred during this session! : restoring previous one!");
XH_uninit;
if ((fexist
(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/old.dat")))
&&
(fexist
(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.ndx")))) {
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/new.dat"));
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/new.ndx"));
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/new.lst"));
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/new.txt"));
rename(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/new.dat"));
rename(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/new.ndx"));
rename(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/new.lst"));
rename(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log),
"hts-cache/new.txt"));
}
opt->state.exit_xh = 2; /* interrupted (no connection detected) */
return 1;
}
// info text
if (cache.txt) {
fclose(cache.txt);
cache.txt = NULL;
}
// purger!
if (cache.lst) {
fclose(cache.lst);
cache.lst = opt->state.strc.lst = NULL;
if (opt->delete_old) {
FILE *old_lst, *new_lst;
//
opt->state._hts_in_html_parsing = 3;
//
old_lst =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/old.lst"), "rb");
if (old_lst) {
const size_t sz =
off_t_to_size_t(fsize(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/new.lst")));
new_lst =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
"hts-cache/new.lst"), "rb");
if (new_lst != NULL && sz != (size_t) -1) {
char *adr = (char *) malloct(sz);
if (adr) {
if (fread(adr, 1, sz, new_lst) == sz) {
char line[1100];
int purge = 0;
while(!feof(old_lst)) {
linput(old_lst, line, 1000);
if (!strstr(adr, line)) { // fichier non trouvé dans le nouveau?
char BIGSTK file[HTS_URLMAXSIZE * 2];
strcpybuff(file, StringBuff(opt->path_html));
strcatbuff(file, line + 1);
file[strlen(file) - 1] = '\0';
if (fexist(file)) { // toujours sur disque: virer
hts_log_print(opt, LOG_INFO, "Purging %s", file);
remove(file);
purge = 1;
}
}
}
{
fseek(old_lst, 0, SEEK_SET);
while(!feof(old_lst)) {
linput(old_lst, line, 1000);
while(strnotempty(line) && (line[strlen(line) - 1] != '/')
&& (line[strlen(line) - 1] != '\\')) {
line[strlen(line) - 1] = '\0';
}
if (strnotempty(line))
line[strlen(line) - 1] = '\0';
if (strnotempty(line))
if (!strstr(adr, line)) { // non trouvé?
char BIGSTK file[HTS_URLMAXSIZE * 2];
strcpybuff(file, StringBuff(opt->path_html));
strcatbuff(file, line + 1);
while((strnotempty(file)) && (rmdir(file) == 0)) { // ok, éliminé (existait)
purge = 1;
if (opt->log) {
hts_log_print(opt, LOG_INFO, "Purging directory %s/",
file);
while(strnotempty(file)
&& (file[strlen(file) - 1] != '/')
&& (file[strlen(file) - 1] != '\\')) {
file[strlen(file) - 1] = '\0';
}
if (strnotempty(file))
file[strlen(file) - 1] = '\0';
}
}
}
}
}
//
if (!purge) {
hts_log_print(opt, LOG_INFO, "No files purged");
}
}
freet(adr);
}
fclose(new_lst);
}
fclose(old_lst);
}
//
opt->state._hts_in_html_parsing = 0;
}
}
// fin purge!
// Indexation
if (opt->kindex)
index_finish(StringBuff(opt->path_html), opt->kindex);
// afficher résumé dans log
if (opt->log != NULL) {
char BIGSTK finalInfo[8192];
int error = fspc(opt, NULL, "error");
int warning = fspc(opt, NULL, "warning");
int info = fspc(opt, NULL, "info");
char BIGSTK htstime[256];
char BIGSTK infoupdated[256];
// int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
LLint n =
(LLint) (HTS_STAT.HTS_TOTAL_RECV /
(max(1, time_local() - HTS_STAT.stat_timestart)));
sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
//sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
infoupdated[0] = '\0';
if (opt->is_update) {
if (HTS_STAT.stat_updated_files > 0) {
sprintf(infoupdated, ", %d files updated",
(int) HTS_STAT.stat_updated_files);
} else {
sprintf(infoupdated, ", no files updated");
}
}
finalInfo[0] = '\0';
sprintf(finalInfo + strlen(finalInfo),
"HTTrack Website Copier/" HTTRACK_VERSION
" mirror complete in %s : " "%d links scanned, %d files written ("
LLintP " bytes overall)%s " "[" LLintP " bytes received at " LLintP
" bytes/sec]", htstime, (int) opt->lien_tot - 1,
(int) HTS_STAT.stat_files, (LLint) HTS_STAT.stat_bytes, infoupdated,
(LLint) HTS_STAT.HTS_TOTAL_RECV, (LLint) n);
if (HTS_STAT.total_packed > 0 && HTS_STAT.total_unpacked > 0) {
int packed_ratio =
(int) ((LLint) (HTS_STAT.total_packed * 100) / HTS_STAT.total_unpacked);
sprintf(finalInfo + strlen(finalInfo),
", " LLintP
" bytes transferred using HTTP compression in %d files, ratio %d%%",
(LLint) HTS_STAT.total_unpacked, HTS_STAT.total_packedfiles,
(int) packed_ratio);
}
if (!opt->nokeepalive && HTS_STAT.stat_sockid > 0
&& HTS_STAT.stat_nrequests > HTS_STAT.stat_sockid) {
int rq = (HTS_STAT.stat_nrequests * 10) / HTS_STAT.stat_sockid;
sprintf(finalInfo + strlen(finalInfo), ", %d.%d requests per connection",
rq / 10, rq % 10);
}
sprintf(finalInfo + strlen(finalInfo), LF);
if (error)
sprintf(finalInfo + strlen(finalInfo),
"(%d errors, %d warnings, %d messages)" LF, error, warning, info);
else
sprintf(finalInfo + strlen(finalInfo),
"(No errors, %d warnings, %d messages)" LF, warning, info);
// Log
fprintf(opt->log, LF "%s", finalInfo);
// Close ZIP
if (cache.zipOutput) {
zipClose(cache.zipOutput, finalInfo);
cache.zipOutput = NULL;
}
}
// fin afficher résumé dans log
// ending
usercommand(opt, 0, NULL, NULL, NULL, NULL);
// désallocation mémoire & buffers
XH_uninit;
return 1; // OK
}
// version 2 pour le reste
// Estimate transfer rate
// a little bit complex, but not too much
/*
.. : idle
^ : event
----|----|----|----|----|----|----|----|---->
1 2 3 4 5 6 7 8 9 time (seconds)
----|----|----|----|----|----|----|----|---->
^........^.........^.........^.........^.... timer 0
----^.........^.........^.........^......... timer 1
0 1 0 1 0 1 0 timer N sets its statistics
* * * * timer 0 resync timer 1
Therefore, each seconds, we resync the transfer rate with 2-seconds
*/
int engine_stats(void) {
#if 0
static FILE *debug_fp = NULL; /* ok */
if (!debug_fp)
debug_fp = fopen("esstat.txt", "wb");
#endif
HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
HTS_STAT.nb = 0;
if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
TStamp cdif = mtime_local();
int i;
for(i = 0; i < 2; i++) {
if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
TStamp dif;
#if 0
fprintf(debug_fp, "set timer %d\n", i);
fflush(debug_fp);
#endif
dif = cdif - HTS_STAT.istat_timestart[i];
if ((TStamp) (dif / 1000) > 0) {
LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
HTS_STAT.rate = (LLint) ((TStamp) ((TStamp) byt / (dif / 1000)));
HTS_STAT.istat_idlasttimer = i; // this timer recently sets the stats
//
HTS_STAT.istat_bytes[i] = HTS_STAT.HTS_TOTAL_RECV;
HTS_STAT.istat_timestart[i] = cdif;
}
return 1; /* refreshed */
}
}
// resynchronization between timer 0 (master) and 1 (slave)
// timer #0 resync timer #1 when reaching 1 second limit
if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
#if 0
fprintf(debug_fp, "resync timer 1\n");
fflush(debug_fp);
#endif
HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
HTS_STAT.istat_timestart[1] = cdif;
HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
}
}
}
return 0;
}
#define _FILTERS (*opt->filters.filters)
#define _FILTERS_PTR (opt->filters.filptr)
#define _ROBOTS ((robots_wizard*)opt->robotsptr)
// bannir host (trop lent etc)
void host_ban(httrackp * opt, int ptr,
struct_back * sback, const char *host) {
lien_back *const back = sback->lnk;
const int back_max = sback->count;
//int l;
int i;
if (host[0] == '!')
return; // erreur.. déja cancellé.. bizarre.. devrait pas arriver
/* sanity check */
if (*_FILTERS_PTR + 1 >= opt->maxfilter) {
opt->maxfilter += HTS_FILTERSINC;
if (filters_init(&_FILTERS, opt->maxfilter, HTS_FILTERSINC) == 0) {
printf("PANIC! : Too many filters : >%d [%d]\n", *_FILTERS_PTR, __LINE__);
hts_log_print(opt, LOG_PANIC, "Too many filters, giving up..(>%d)",
*_FILTERS_PTR);
hts_log_print(opt, LOG_INFO,
"To avoid that: use #F option for more filters (example: -#F5000)");
assertf("too many filters - giving up" == NULL);
}
}
// interdire host
assertf((*_FILTERS_PTR) < opt->maxfilter);
if (*_FILTERS_PTR < opt->maxfilter) {
strcpybuff(_FILTERS[*_FILTERS_PTR], "-");
strcatbuff(_FILTERS[*_FILTERS_PTR], host);
strcatbuff(_FILTERS[*_FILTERS_PTR], "/*"); // host/ * interdit
(*_FILTERS_PTR)++;
}
// oups
if (strlen(host) <= 1) { // euhh?? longueur <= 1
if (strcmp(host, "file://")) {
//## if (host[0]!=lOCAL_CHAR) { // pas local
hts_log_print(opt, LOG_PANIC,
"PANIC! HostCancel detected memory leaks [char %d]",
host[0]);
return; // purée
}
}
// couper connexion
for(i = 0; i < back_max; i++) {
if (back[i].status >= 0) // réception OU prêt
if (strfield2(back[i].url_adr, host)) {
#if HTS_DEBUG_CLOSESOCK
DEBUG_W("host control: deletehttp\n");
#endif
back[i].status = 0; // terminé
back_set_finished(sback, i);
if (back[i].r.soc != INVALID_SOCKET)
deletehttp(&back[i].r);
back[i].r.soc = INVALID_SOCKET;
back[i].r.statuscode = STATUSCODE_TIMEOUT; // timeout (peu importe si c'est un traffic jam)
strcpybuff(back[i].r.msg, "Link Cancelled by host control");
hts_log_print(opt, LOG_DEBUG, "Shutdown: %s%s", back[i].url_adr,
back[i].url_fil);
}
}
// effacer liens
//l=strlen(host);
for(i = 0; i < opt->lien_tot; i++) {
//if (heap(i)->adr_len==l) { // même taille de chaîne
// Calcul de taille sécurisée
if (heap(i)) {
if (heap(i)->adr) {
int l = 0;
while((heap(i)->adr[l]) && (l < 1020))
l++;
if ((l > 0) && (l < 1020)) { // sécurité
if (strfield2(jump_identification_const(heap(i)->adr), host)) { // host
hts_log_print(opt, LOG_DEBUG, "Cancel: %s%s", heap(i)->adr,
heap(i)->fil);
hts_invalidate_link(opt, i); // invalidate hashtable entry
// on efface pas le hash, because si on rencontre le lien, reverif sav..
}
} else {
if (opt->log != NULL) {
char dmp[1040];
dmp[0] = '\0';
strncatbuff(dmp, heap(i)->adr, 1024);
hts_log_print(opt, LOG_WARNING,
"WARNING! HostCancel detected memory leaks [len %d at %d]",
l, i);
hts_log_print(opt, LOG_WARNING,
"dump 1024 bytes (address %p): " LF "%s",
heap(i)->adr, dmp);
}
}
} else {
hts_log_print(opt, LOG_WARNING,
"WARNING! HostCancel detected memory leaks [adr at %d]",
i);
}
} else {
hts_log_print(opt, LOG_WARNING,
"WARNING! HostCancel detected memory leaks [null at %d]",
i);
}
//}
}
}
int filters_init(char ***ptrfilters, int maxfilter, int filterinc) {
char **filters = *ptrfilters;
int filter_max = maximum(maxfilter, 128);
if (filters == NULL) {
filters = (char **) malloct(sizeof(char *) * (filter_max + 2));
memset(filters, 0, sizeof(char *) * (filter_max + 2)); // filters[0] == 0
} else {
filters = (char **) realloct(filters, sizeof(char *) * (filter_max + 2));
}
if (filters) {
if (filters[0] == NULL) {
filters[0] =
(char *) malloct(sizeof(char) * (filter_max + 2) *
(HTS_URLMAXSIZE * 2));
memset(filters[0], 0,
sizeof(char) * (filter_max + 2) * (HTS_URLMAXSIZE * 2));
} else {
filters[0] =
(char *) realloct(filters[0],
sizeof(char) * (filter_max +
2) * (HTS_URLMAXSIZE * 2));
}
if (filters[0] == NULL) {
freet(filters);
filters = NULL;
}
}
if (filters != NULL) {
int i;
int from;
if (filterinc == 0)
from = 0;
else
from = filter_max - filterinc;
for(i = 0; i <= filter_max; i++) { // PLUS UN (sécurité)
filters[i] = filters[0] + i * (HTS_URLMAXSIZE * 2);
}
for(i = from; i <= filter_max; i++) { // PLUS UN (sécurité)
filters[i][0] = '\0'; // clear
}
}
*ptrfilters = filters;
return (filters != NULL) ? filter_max : 0;
}
static int mkdir_compat(const char *pathname) {
#ifdef _WIN32
return mkdir(pathname);
#else
return mkdir(pathname, HTS_ACCESS_FOLDER);
#endif
}
/* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */
/* Note: preserve errno */
HTSEXT_API int dir_exists(const char *path) {
const int err = errno;
STRUCT_STAT st;
char BIGSTK file[HTS_URLMAXSIZE * 2];
int i = 0;
if (strnotempty(path) == 0) {
return 0;
}
if (strlen(path) > HTS_URLMAXSIZE) {
return 0;
}
/* Get a copy */
strcpybuff(file, path);
#ifdef _WIN32
/* To system name */
for(i = 0; file[i] != 0; i++) {
if (file[i] == '/') {
file[i] = PATH_SEPARATOR;
}
}
#endif
/* Get prefix (note: file can not be empty here) */
for(i = (int) strlen(file) - 1; i > 0 && file[i] != PATH_SEPARATOR; i--) ;
for(; i > 0 && file[i] == PATH_SEPARATOR; i--) ;
file[i + 1] = '\0';
/* Check the final dir */
if (STAT(file, &st) == 0 && S_ISDIR(st.st_mode)) {
return 1; /* EXISTS */
}
errno = err;
return 0; /* DOES NOT EXIST */
}
/* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */
/* Note: *not* UTF-8 */
HTSEXT_API int structcheck(const char *path) {
struct stat st;
char BIGSTK tmpbuf[HTS_URLMAXSIZE * 2];
char BIGSTK file[HTS_URLMAXSIZE * 2];
int i = 0;
int npaths;
if (strnotempty(path) == 0)
return 0;
if (strlen(path) > HTS_URLMAXSIZE) {
errno = EINVAL;
return -1;
}
/* Get a copy */
strcpybuff(file, path);
#ifdef _WIN32
/* To system name */
for(i = 0; file[i] != 0; i++) {
if (file[i] == '/') {
file[i] = PATH_SEPARATOR;
}
}
#endif
/* Get prefix (note: file can not be empty here) */
for(i = (int) strlen(file) - 1; i > 0 && file[i] != PATH_SEPARATOR; i--) ;
for(; i > 0 && file[i] == PATH_SEPARATOR; i--) ;
file[i + 1] = '\0';
/* First check the final dir */
if (stat(file, &st) == 0 && S_ISDIR(st.st_mode)) {
return 0; /* OK */
}
/* Start from the beginning */
i = 0;
/* Skip irrelevant part (the root slash, or the drive path) */
#ifdef _WIN32
if (file[0] != 0 && file[1] == ':') { /* f:\ */
i += 2;
if (file[i] == PATH_SEPARATOR) { /* f:\ */
i++;
}
} else if (file[0] == PATH_SEPARATOR && file[1] == PATH_SEPARATOR) { /* \\mch */
int j;
i += 2;
// skip machine and first dir (share) or we'll have troubles (TomZ)
for(j = 0; j < 2; j++) {
// skip segment
for(; file[i] != '\0' && file[i] != PATH_SEPARATOR; i++) ;
// skip final separator
if (file[i] == PATH_SEPARATOR) { /* f:\ */
i++;
}
}
}
#endif
/* Check paths */
for(npaths = 1;; npaths++) {
char end_char;
/* Go to next path */
/* Skip separator(s) */
for(; file[i] == PATH_SEPARATOR; i++) ;
/* Next separator */
for(; file[i] != 0 && file[i] != PATH_SEPARATOR; i++) ;
/* Check */
end_char = file[i];
if (end_char != 0) {
file[i] = '\0';
}
if (stat(file, &st) == 0) { /* Something exists */
if (!S_ISDIR(st.st_mode)) {
#if HTS_REMOVE_ANNOYING_INDEX
if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */
sprintf(tmpbuf, "%s.txt", file);
if (rename(file, tmpbuf) != 0) { /* Can't rename regular file */
return -1;
}
if (mkdir_compat(file) != 0) { /* Can't create directory */
return -1;
}
}
#else
#error Not implemented
#endif
}
} else { /* Nothing exists ; create directory */
if (mkdir_compat(file) != 0) { /* Can't create directory */
return -1;
}
}
if (end_char == 0) { /* End */
break;
} else {
file[i] = end_char; /* Restore / */
}
}
return 0;
}
/* path must end with "/" or with the finename (/tmp/bar/ or /tmp/bar/foo.zip) */
/* Note: UTF-8 */
HTSEXT_API int structcheck_utf8(const char *path) {
STRUCT_STAT st;
char BIGSTK tmpbuf[HTS_URLMAXSIZE * 2];
char BIGSTK file[HTS_URLMAXSIZE * 2];
int i = 0;
int npaths;
if (strnotempty(path) == 0)
return 0;
if (strlen(path) > HTS_URLMAXSIZE) {
errno = EINVAL;
return -1;
}
/* Get a copy */
strcpybuff(file, path);
#ifdef _WIN32
/* To system name */
for(i = 0; file[i] != 0; i++) {
if (file[i] == '/') {
file[i] = PATH_SEPARATOR;
}
}
#endif
/* Get prefix (note: file can not be empty here) */
for(i = (int) strlen(file) - 1; i > 0 && file[i] != PATH_SEPARATOR; i--) ;
for(; i > 0 && file[i] == PATH_SEPARATOR; i--) ;
file[i + 1] = '\0';
/* First check the final dir */
if (STAT(file, &st) == 0 && S_ISDIR(st.st_mode)) {
return 0; /* OK */
}
/* Start from the beginning */
i = 0;
/* Skip irrelevant part (the root slash, or the drive path) */
#ifdef _WIN32
if (file[0] != 0 && file[1] == ':') { /* f:\ */
i += 2;
if (file[i] == PATH_SEPARATOR) { /* f:\ */
i++;
}
} else if (file[0] == PATH_SEPARATOR && file[1] == PATH_SEPARATOR) { /* \\mch */
int j;
i += 2;
// skip machine and first dir (share) or we'll have troubles (TomZ)
for(j = 0; j < 2; j++) {
// skip segment
for(; file[i] != '\0' && file[i] != PATH_SEPARATOR; i++) ;
// skip final separator
if (file[i] == PATH_SEPARATOR) { /* f:\ */
i++;
}
}
}
#endif
/* Check paths */
for(npaths = 1;; npaths++) {
char end_char;
/* Go to next path */
/* Skip separator(s) */
for(; file[i] == PATH_SEPARATOR; i++) ;
/* Next separator */
for(; file[i] != 0 && file[i] != PATH_SEPARATOR; i++) ;
/* Check */
end_char = file[i];
if (end_char != 0) {
file[i] = '\0';
}
if (STAT(file, &st) == 0) { /* Something exists */
if (!S_ISDIR(st.st_mode)) {
#if HTS_REMOVE_ANNOYING_INDEX
if (S_ISREG(st.st_mode)) { /* Regular file in place ; move it and create directory */
sprintf(tmpbuf, "%s.txt", file);
if (RENAME(file, tmpbuf) != 0) { /* Can't rename regular file */
return -1;
}
if (MKDIR(file) != 0) { /* Can't create directory */
return -1;
}
}
#else
#error Not implemented
#endif
}
} else { /* Nothing exists ; create directory */
if (MKDIR(file) != 0) { /* Can't create directory */
return -1;
}
}
if (end_char == 0) { /* End */
break;
} else {
file[i] = end_char; /* Restore / */
}
}
return 0;
}
// sauver un fichier
int filesave(httrackp * opt, const char *adr, int len, const char *s,
const char *url_adr, const char *url_fil) {
FILE *fp;
// écrire le fichier
if ((fp = filecreate(&opt->state.strc, s)) != NULL) {
int nl = 0;
if (len > 0) {
nl = (int) fwrite(adr, 1, len, fp);
}
fclose(fp);
if (nl != len) // erreur
return -1;
} else
return -1;
return 0;
}
/* We should stop */
int check_fatal_io_errno(void) {
switch (errno) {
#ifdef EMFILE
case EMFILE: /* Too many open files */
#endif
#ifdef ENOSPC
case ENOSPC: /* No space left on device */
#endif
#ifdef EROFS
case EROFS: /* Read-only file system */
#endif
return 1;
break;
}
return 0;
}
// ouvrir un fichier (avec chemin Un*x)
/* Note: utf-8 */
FILE *filecreate(filenote_strc * strc, const char *s) {
char BIGSTK fname[HTS_URLMAXSIZE * 2];
FILE *fp;
int last_errno = 0;
fname[0] = '\0';
// noter lst
if (strc != NULL) {
filenote(strc, s, NULL);
}
strcpybuff(fname, s);
#if HTS_DOSNAME
// remplacer / par des slash arrière
{
int i = 0;
while(fname[i]) {
if (fname[i] == '/')
fname[i] = '\\';
i++;
}
}
// a partir d'ici le slash devient antislash
#endif
/* Try to open the file */
fp = FOPEN(fname, "wb");
/* Error ? Check the directory structure and retry. */
if (fp == NULL) {
last_errno = errno;
if (structcheck_utf8(s) != 0) {
last_errno = errno;
} else {
last_errno = 0;
}
fp = FOPEN(fname, "wb");
}
if (fp == NULL && last_errno != 0) {
errno = last_errno;
}
#ifndef _WIN32
if (fp != NULL)
chmod(fname, HTS_ACCESS_FILE);
#endif
return fp;
}
// ouvrir un fichier (avec chemin Un*x)
FILE *fileappend(filenote_strc * strc, const char *s) {
char BIGSTK fname[HTS_URLMAXSIZE * 2];
FILE *fp;
fname[0] = '\0';
// noter lst
filenote(strc, s, NULL);
// if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s); // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
strcpybuff(fname, s);
#if HTS_DOSNAME
// remplacer / par des slash arrière
{
int i = 0;
while(fname[i]) {
if (fname[i] == '/')
fname[i] = '\\';
i++;
}
}
// a partir d'ici le slash devient antislash
#endif
// ouvrir
fp = FOPEN(fname, "ab");
#ifndef _WIN32
if (fp != NULL)
chmod(fname, HTS_ACCESS_FILE);
#endif
return fp;
}
// create an empty file
int filecreateempty(filenote_strc * strc, const char *filename) {
FILE *fp;
fp = filecreate(strc, filename); // filenote & co
if (fp) {
fclose(fp);
return 1;
} else
return 0;
}
// noter fichier
int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
// gestion du fichier liste liste
if (params) {
//filecreate_params* p = (filecreate_params*) params;
strcpybuff(strc->path, params->path);
strc->lst = params->lst;
return 0;
} else if (strc->lst) {
char BIGSTK savelst[HTS_URLMAXSIZE * 2];
char catbuff[CATBUFF_SIZE];
strcpybuff(savelst, fslash(catbuff, sizeof(catbuff), s));
// couper chemin?
if (strnotempty(strc->path)) {
if (strncmp(fslash(catbuff, sizeof(catbuff), strc->path), savelst, strlen(strc->path)) == 0) { // couper
strcpybuff(savelst, s + strlen(strc->path));
}
}
fprintf(strc->lst, "[%s]" LF, savelst);
fflush(strc->lst);
}
return 1;
}
/* Note: utf-8 */
void file_notify(httrackp * opt, const char *adr, const char *fil,
const char *save, int create, int modify, int not_updated) {
RUN_CALLBACK6(opt, filesave2, adr, fil, save, create, modify, not_updated);
}
// executer commande utilisateur
static void postprocess_file(httrackp * opt, const char *save, const char *adr,
const char *fil);
void usercommand(httrackp * opt, int _exe, const char *_cmd, const char *file,
const char *adr, const char *fil) {
usercommand_strc *strc = &opt->state.usercmd;
/* Callback */
if (_exe) {
strcpybuff(strc->cmd, _cmd);
if (strnotempty(strc->cmd))
strc->exe = _exe;
else
strc->exe = 0;
}
/* post-processing */
postprocess_file(opt, file, adr, fil);
if (file != NULL && strnotempty(file)) {
RUN_CALLBACK1(opt, filesave, file);
}
if (strc->exe) {
if (file != NULL && strnotempty(file)) {
if (strnotempty(strc->cmd)) {
usercommand_exe(strc->cmd, file);
}
}
}
}
void usercommand_exe(const char *cmd, const char *file) {
char BIGSTK temp[8192];
size_t i;
temp[0] = '\0';
//
for(i = 0; cmd[i] != '\0'; i++) {
if ((cmd[i] == '$') && (cmd[i + 1] == '0')) {
strcatbuff(temp, file);
i++;
} else {
char c[2];
c[0] = cmd[i];
c[1] = '\0';
strcatbuff(temp, c);
}
}
if (system(temp) == -1) {
assertf(!"can not spawn process");
}
}
static void postprocess_file(httrackp * opt, const char *save, const char *adr,
const char *fil) {
//int first = 0;
/* MIME-html archive to build */
if (opt != NULL && opt->mimehtml) {
if (adr != NULL && strcmp(adr, "primary") == 0) {
adr = NULL;
}
if (save != NULL && opt != NULL && adr != NULL && adr[0]
&& strnotempty(save) && fexist(save)) {
const char *rsc_save = save;
const char *rsc_fil = strrchr(fil, '/');
int n;
if (rsc_fil == NULL)
rsc_fil = fil;
if (strncmp
(fslash(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save),
fslash(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_html_utf8)), (n =
(int)
strlen
(StringBuff
(opt->
path_html_utf8))))
== 0) {
rsc_save += n;
}
if (!opt->state.mimehtml_created) {
//first = 1;
opt->state.mimefp =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_html), "index.mht"),
"wb");
(void) unlink(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_html),
"index.eml"));
#ifndef _WIN32
if (symlink("index.mht",
fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html),
"index.eml")) != 0) {
if (errno != EPERM) {
hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
"could not create symbolic link from index.mht to index.eml");
}
}
#endif
if (opt->state.mimefp != NULL) {
char BIGSTK rndtmp[1024], currtime[256];
srand((unsigned int) time(NULL));
time_gmt_rfc822(currtime);
sprintf(rndtmp, "%d_%d", (int) time(NULL), (int) rand());
StringRoom(opt->state.mimemid, 256);
sprintf(StringBuffRW(opt->state.mimemid), "----=_MIMEPart_%s_=----",
rndtmp);
StringSetLength(opt->state.mimemid, -1);
fprintf(opt->state.mimefp,
"From: HTTrack Website Copier <nobody@localhost>\r\n"
"Subject: Local mirror\r\n" "Date: %s\r\n"
"Message-ID: <httrack_%s@localhost>\r\n"
"Content-Type: multipart/related;\r\n"
"\tboundary=\"%s\";\r\n" "\ttype=\"text/html\"\r\n"
"MIME-Version: 1.0\r\n"
"\r\nThis message is a RFC MIME-compliant multipart message.\r\n"
"\r\n", currtime, rndtmp, StringBuff(opt->state.mimemid));
opt->state.mimehtml_created = 1;
} else {
opt->state.mimehtml_created = -1;
hts_log_print(opt, LOG_ERROR, "unable to create index.mht");
}
}
if (opt->state.mimehtml_created == 1 && opt->state.mimefp != NULL) {
FILE *fp = FOPEN(save, "rb");
if (fp != NULL) {
char buff[60 * 100 + 2];
char mimebuff[256];
char BIGSTK cid[HTS_URLMAXSIZE * 3];
size_t len;
int isHtml = (ishtml(opt, save) == 1);
mimebuff[0] = '\0';
/* CID */
make_content_id(adr, fil, cid, sizeof(cid));
guess_httptype(opt, mimebuff, save);
fprintf(opt->state.mimefp, "--%s\r\n",
StringBuff(opt->state.mimemid));
/*if (first)
fprintf(opt->state.mimefp, "Content-disposition: inline\r\n");
else */
fprintf(opt->state.mimefp,
"Content-disposition: attachment; filename=\"%s\"\r\n",
rsc_save);
fprintf(opt->state.mimefp,
"Content-Type: %s\r\n" "Content-Transfer-Encoding: %s\r\n"
/*"Content-Location: http://localhost/%s\r\n" */
"Content-ID: <%s>\r\n" "\r\n", mimebuff,
isHtml ? "8bit" : "base64"
/*, rsc_save */
, cid);
while((len = fread(buff, 1, sizeof(buff) - 2, fp)) > 0) {
buff[len] = '\0';
if (!isHtml) {
char base64buff[60 * 100 * 2];
code64((unsigned char *) buff, (int) len,
(unsigned char *) base64buff, 1);
fprintf(opt->state.mimefp, "%s", base64buff);
} else {
fprintf(opt->state.mimefp, "%s", buff);
}
}
fclose(fp);
fprintf(opt->state.mimefp, "\r\n\r\n");
}
}
} else if (save == NULL) {
if (opt->state.mimehtml_created == 1 && opt->state.mimefp != NULL) {
fprintf(opt->state.mimefp, "--%s--\r\n",
StringBuff(opt->state.mimemid));
fclose(opt->state.mimefp);
opt->state.mimefp = NULL;
}
}
}
}
// écrire n espaces dans fp
int fspc(httrackp * opt, FILE * fp, const char *type) {
fspc_strc *const strc = (opt != NULL) ? &opt->state.fspc : NULL;
if (fp != NULL) {
char s[256];
time_t tt;
struct tm *A;
tt = time(NULL);
A = localtime(&tt);
if (A == NULL) {
int localtime_returned_null = 0;
assertf(localtime_returned_null);
}
strftime(s, 250, "%H:%M:%S", A);
if (strnotempty(type))
fprintf(fp, "%s\t%c%s: \t", s, hichar(*type), type + 1);
else
fprintf(fp, "%s\t \t", s);
if (strc != NULL) {
if (strcmp(type, "warning") == 0)
strc->warning++;
else if (strcmp(type, "error") == 0)
strc->error++;
else if (strcmp(type, "info") == 0)
strc->info++;
}
} else if (strc == NULL) {
return 0;
} else if (!type) {
strc->error = strc->warning = strc->info = 0; // reset
} else if (strcmp(type, "warning") == 0)
return strc->warning;
else if (strcmp(type, "error") == 0)
return strc->error;
else if (strcmp(type, "info") == 0)
return strc->info;
return 0;
}
// vérifier taux de transfert
#if 0
void check_rate(TStamp stat_timestart, int maxrate) {
// vérifier taux de transfert (pas trop grand?)
/*
if (maxrate>0) {
int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart)); // taux actuel de transfert
HTS_STAT.HTS_TOTAL_RECV_STATE=0;
if (r>maxrate) { // taux>taux autorisé
int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
if (taux<15)
HTS_STAT.HTS_TOTAL_RECV_STATE=1; // ralentir un peu (<15% dépassement)
else if (taux<50)
HTS_STAT.HTS_TOTAL_RECV_STATE=2; // beaucoup (<50% dépassement)
else
HTS_STAT.HTS_TOTAL_RECV_STATE=3; // énormément (>50% dépassement)
}
}
*/
}
#endif
// ---
// sous routines liées au moteur et au backing
// supplemental links ready (done) after ptr or ready in background
int backlinks_done(const struct_back * sback,
lien_url ** liens, int lien_tot,
int ptr) {
int n = 0;
#if 0
int i;
//Links done and stored in cache
for(i = ptr + 1; i < lien_tot; i++) {
if (heap(i)) {
if (heap(i)->pass2 == -1) {
n++;
}
}
}
#else
// finalized in background
n += HTS_STAT.stat_background;
#endif
n += back_done_incache(sback);
return n;
}
// remplir backing si moins de max_bytes en mémoire
HTS_INLINE int back_fillmax(struct_back * sback, httrackp * opt,
cache_back * cache, int ptr,
int numero_passe) {
if (!opt->state.stop) {
if (back_incache(sback) < opt->maxcache) { // pas trop en mémoire?
return back_fill(sback, opt, cache, ptr, numero_passe);
}
}
return -1; /* plus de place */
}
int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt) {
int n = opt->maxsoc - back_nsoc(sback);
// connect limiter
if (n > 0 && opt->maxconn > 0 && HTS_STAT.last_connect > 0) {
TStamp opTime =
HTS_STAT.last_request ? HTS_STAT.last_request : HTS_STAT.last_connect;
TStamp cTime = mtime_local();
TStamp lap = (cTime - opTime);
TStamp minLap = (TStamp) (1000.0 / opt->maxconn);
if (lap < minLap) {
n = 0;
} else if (minLap != 0) {
int nMax = (int) (lap / minLap);
n = min(n, nMax);
}
}
return n;
}
int back_pluggable_sockets(struct_back * sback, httrackp * opt) {
int n;
// ajouter autant de socket qu'on peut ajouter
n = back_pluggable_sockets_strict(sback, opt);
// vérifier qu'il restera assez de place pour les tests ensuite (en théorie, 1 entrée libre restante suffirait)
n = min(n, back_available(sback) - 8);
// no space left on backing stack - do not back anymore
if (back_stack_available(sback) <= 2)
n = 0;
return n;
}
// remplir backing
int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
int ptr, int numero_passe) {
int n = back_pluggable_sockets(sback, opt);
if (opt->savename_delayed == 2 && !opt->delayed_cached) /* cancel (always delayed) */
return 0;
if (n > 0) {
int p;
if (ptr < cache->ptr_last) { /* restart (2 scans: first html, then non html) */
cache->ptr_ant = 0;
}
p = ptr + 1;
/* on a déja parcouru */
if (p < cache->ptr_ant)
p = cache->ptr_ant;
while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
//while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
int ok = 1;
// on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
if (heap(p)->pass2) { // 2è passe
if (numero_passe != 1)
ok = 0;
} else {
if (numero_passe != 0)
ok = 0;
}
// Why in hell did I do that ?
//if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0'
// && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0) // lookup in liens_record
//{
// ok = 0;
//}
if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
|| hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
ok = 0;
}
// note: si un backing est fini, il reste en mémoire jusqu'à ce que
// le ptr l'atteigne
if (ok) {
if (!back_exist
(sback, opt, heap(p)->adr, heap(p)->fil, heap(p)->sav)) {
if (back_add
(sback, opt, cache, heap(p)->adr, heap(p)->fil, heap(p)->sav,
heap(heap(p)->precedent)->adr, heap(heap(p)->precedent)->fil,
heap(p)->testmode) == -1) {
hts_log_print(opt, LOG_DEBUG,
"error: unable to add more links through back_add for back_fill");
#if BDEBUG==1
printf("error while adding\n");
#endif
n = 0; // sortir
} else {
n--;
#if BDEBUG==1
printf("backing: %s%s\n", heap(p)->adr, heap(p)->fil);
#endif
}
}
}
p++;
} // while
/* sauver position dernière anticipation */
cache->ptr_ant = p;
cache->ptr_last = ptr;
}
return 0;
}
// ---
// Poll stdin.. si besoin
#if HTS_POLL
// lecture stdin des caractères disponibles
int read_stdin(char *s, int max) {
int i = 0;
while((check_stdin()) && (i < (max - 1)))
s[i++] = fgetc(stdin);
s[i] = '\0';
return i;
}
#ifdef _WIN32
int check_stdin(void) {
return (_kbhit());
}
#else
int check_flot(T_SOC s) {
fd_set fds;
struct timeval tv;
FD_ZERO(&fds);
FD_SET((T_SOC) s, &fds);
tv.tv_sec = 0;
tv.tv_usec = 0;
select(s + 1, &fds, NULL, NULL, &tv);
return FD_ISSET(s, &fds);
}
int check_stdin(void) {
fflush(stdout);
fflush(stdin);
if (check_flot(0))
return 1;
return 0;
}
#endif
#endif
int check_sockerror(T_SOC s) {
fd_set fds;
struct timeval tv;
FD_ZERO(&fds);
FD_SET((T_SOC) s, &fds);
tv.tv_sec = 0;
tv.tv_usec = 0;
select((int) s + 1, NULL, NULL, &fds, &tv);
return FD_ISSET(s, &fds);
}
/* check incoming data */
int check_sockdata(T_SOC s) {
fd_set fds;
struct timeval tv;
FD_ZERO(&fds);
FD_SET((T_SOC) s, &fds);
tv.tv_sec = 0;
tv.tv_usec = 0;
select((int) s + 1, &fds, NULL, NULL, &tv);
return FD_ISSET(s, &fds);
}
// Attente de touche
int ask_continue(httrackp * opt) {
const char *s;
s = RUN_CALLBACK1(opt, query2, opt->state.HTbuff);
if (s) {
if (strnotempty(s)) {
if ((strfield2(s, "N")) || (strfield2(s, "NO")) || (strfield2(s, "NON")))
return 0;
}
return 1;
}
return 1;
}
// nombre de digits dans un nombre
int nombre_digit(int n) {
int i = 1;
while(n >= 10) {
n /= 10;
i++;
}
return i;
}
// renvoi adresse de la fin du token dans p
// renvoi NULL si la chaine est un token unique
// (PATCHE également la chaine)
// ex: "test" "test2" renvoi adresse sur espace
// flag==1 si chaine comporte des echappements comme \"
char *next_token(char *p, int flag) {
int detect = 0;
int quote = 0;
p--;
do {
p++;
if (flag && (*p == '\\')) { // sauter \x ou \"
if (quote) {
char c = '\0';
if (*(p + 1) == '\\')
c = '\\';
else if (*(p + 1) == '"')
c = '"';
if (c) {
char BIGSTK tempo[8192];
tempo[0] = c;
tempo[1] = '\0';
strcatbuff(tempo, p + 2);
strcpybuff(p, tempo);
}
}
} else if (*p == 34) { // guillemets (de fin)
char BIGSTK tempo[8192];
tempo[0] = '\0';
strcatbuff(tempo, p + 1);
strcpybuff(p, tempo); /* wipe "" */
p--;
/* */
quote = !quote;
} else if (*p == 32) {
if (!quote)
detect = 1;
} else if (*p == '\0') {
p = NULL;
detect = 1;
}
} while(!detect);
return p;
}
static int hts_cancel_file_push_(httrackp * opt, const char *url) {
if (url != NULL && url[0] != '\0') {
htsoptstatecancel **cancel;
/* search for available place to store a new htsoptstatecancel* */
for(cancel = &opt->state.cancel; *cancel != NULL;
cancel = &((*cancel)->next)) {
if (strcmp((*cancel)->url, url) == 0) {
return 1; /* already there */
}
}
*cancel = malloct(sizeof(htsoptstatecancel));
(*cancel)->next = NULL;
(*cancel)->url = strdupt(url);
return 0;
}
return 1;
}
/* cancel a file (locked) */
HTSEXT_API int hts_cancel_file_push(httrackp * opt, const char *url) {
int ret;
hts_mutexlock(&opt->state.lock);
ret = hts_cancel_file_push_(opt, url);
hts_mutexrelease(&opt->state.lock);
return ret;
}
static char *hts_cancel_file_pop_(httrackp * opt) {
if (opt->state.cancel != NULL) {
htsoptstatecancel **cancel;
htsoptstatecancel *ret;
for(cancel = &opt->state.cancel; (*cancel)->next != NULL;
cancel = &((*cancel)->next)) ;
ret = *cancel;
*cancel = NULL;
return ret->url;
}
return NULL; /* no entry */
}
char *hts_cancel_file_pop(httrackp * opt) {
char *ret;
hts_mutexlock(&opt->state.lock);
ret = hts_cancel_file_pop_(opt);
hts_mutexrelease(&opt->state.lock);
return ret;
}
HTSEXT_API void hts_cancel_test(httrackp * opt) {
if (opt->state._hts_in_html_parsing == 2)
opt->state._hts_cancel = 2;
}
HTSEXT_API void hts_cancel_parsing(httrackp * opt) {
if (opt->state._hts_in_html_parsing)
opt->state._hts_cancel = 1;
}
// en train de parser un fichier html? réponse: % effectués
// flag>0 : refresh demandé
HTSEXT_API int hts_is_parsing(httrackp * opt, int flag) {
if (opt->state._hts_in_html_parsing) { // parsing?
if (flag >= 0)
opt->state._hts_in_html_poll = 1; // faudrait un tit refresh
return max(opt->state._hts_in_html_done, 1); // % effectués
} else {
return 0; // non
}
}
HTSEXT_API int hts_is_testing(httrackp * opt) { // 0 non 1 test 2 purge
if (opt->state._hts_in_html_parsing == 2)
return 1;
else if (opt->state._hts_in_html_parsing == 3)
return 2;
else if (opt->state._hts_in_html_parsing == 4)
return 3;
else if (opt->state._hts_in_html_parsing == 5) // scheduling
return 4;
else if (opt->state._hts_in_html_parsing == 6) // wait for slot
return 5;
return 0;
}
HTSEXT_API int hts_is_exiting(httrackp * opt) {
return opt->state.exit_xh;
}
// message d'erreur?
char *hts_errmsg(httrackp * opt) {
return opt->state._hts_errmsg;
}
// mode pause transfer
HTSEXT_API int hts_setpause(httrackp * opt, int p) {
if (p >= 0)
opt->state._hts_setpause = p;
return opt->state._hts_setpause;
}
// ask for termination
HTSEXT_API int hts_request_stop(httrackp * opt, int force) {
if (opt != NULL) {
hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
hts_mutexlock(&opt->state.lock);
opt->state.stop = 1;
hts_mutexrelease(&opt->state.lock);
}
return 0;
}
HTSEXT_API int hts_has_stopped(httrackp * opt) {
int ended;
hts_mutexlock(&opt->state.lock);
ended = opt->state.is_ended;
hts_mutexrelease(&opt->state.lock);
return ended;
}
// régler en cours de route les paramètres réglables..
// -1 : erreur
//HTSEXT_API int hts_setopt(httrackp* set_opt) {
// if (set_opt) {
// httrackp* engine_opt=hts_declareoptbuffer(NULL);
// if (engine_opt) {
// //_hts_setopt=opt;
// copy_htsopt(set_opt,engine_opt);
// }
// }
// return 0;
//}
// ajout d'URL
// -1 : erreur
HTSEXT_API int hts_addurl(httrackp * opt, char **url) {
if (url)
opt->state._hts_addurl = url;
return (opt->state._hts_addurl != NULL);
}
HTSEXT_API int hts_resetaddurl(httrackp * opt) {
opt->state._hts_addurl = NULL;
return (opt->state._hts_addurl != NULL);
}
// copier nouveaux paramètres si besoin
HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
if (from->maxsite > -1)
to->maxsite = from->maxsite;
if (from->maxfile_nonhtml > -1)
to->maxfile_nonhtml = from->maxfile_nonhtml;
if (from->maxfile_html > -1)
to->maxfile_html = from->maxfile_html;
if (from->maxsoc > 0)
to->maxsoc = from->maxsoc;
if (from->nearlink > -1)
to->nearlink = from->nearlink;
if (from->timeout > -1)
to->timeout = from->timeout;
if (from->rateout > -1)
to->rateout = from->rateout;
if (from->maxtime > -1)
to->maxtime = from->maxtime;
if (from->maxrate > -1)
to->maxrate = from->maxrate;
if (from->maxconn > 0)
to->maxconn = from->maxconn;
if (StringNotEmpty(from->user_agent))
StringCopyS(to->user_agent, from->user_agent);
if (from->retry > -1)
to->retry = from->retry;
if (from->hostcontrol > -1)
to->hostcontrol = from->hostcontrol;
if (from->errpage > -1)
to->errpage = from->errpage;
if (from->parseall > -1)
to->parseall = from->parseall;
// test all: bit 8 de travel
if (from->travel > -1) {
if (from->travel & 256)
to->travel |= 256;
else
to->travel &= 255;
}
return 0;
}
//
/* External modules callback */
int htsAddLink(htsmoduleStruct * str, char *link) {
if (link != NULL && str != NULL && link[0] != '\0') {
ENGINE_LOAD_CONTEXT_BASE();
/* */
lien_adrfilsave afs;
char BIGSTK codebase[HTS_URLMAXSIZE * 2];
/* */
int pass_fix, prio_fix;
/* */
int forbidden_url = 1;
codebase[0] = '\0';
hts_log_print(opt, LOG_DEBUG, "(module): adding link : '%s'", link);
// recopie de "creer le lien"
//
if (!RUN_CALLBACK1(opt, linkdetected, link)) {
hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper",
link);
return 0;
}
if (!RUN_CALLBACK2(opt, linkdetected2, link, NULL)) {
hts_log_print(opt, LOG_ERROR, "Link %s refused by external wrapper(2)",
link);
return 0;
}
// adr = c'est la même
// fil et save: save2 et fil2
prio_fix = maximum(heap(ptr)->depth - 1, 0);
pass_fix = max(heap(ptr)->pass2, numero_passe);
if (heap(ptr)->cod)
strcpybuff(codebase, heap(ptr)->cod); // codebase valable pour tt les classes descendantes
if (strnotempty(codebase) == 0) { // pas de codebase, construire
char *a;
if (str->relativeToHtmlLink == 0)
strcpybuff(codebase, heap(ptr)->fil);
else
strcpybuff(codebase, heap(heap(ptr)->precedent)->fil);
a = codebase + strlen(codebase) - 1;
while((*a) && (*a != '/') && (a > codebase))
a--;
if (*a == '/')
*(a + 1) = '\0'; // couper
} else { // couper http:// éventuel
if (strfield(codebase, "http://")) {
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
char *a = codebase + 7;
a = strchr(a, '/'); // après host
if (a) { // ** msg erreur et vérifier?
strcpybuff(tempo, a);
strcpybuff(codebase, tempo); // couper host
} else {
hts_log_print(opt, LOG_ERROR, "Unexpected strstr error in base %s",
codebase);
}
}
}
if (!((int) strlen(codebase) < HTS_URLMAXSIZE)) { // trop long
hts_log_print(opt, LOG_ERROR, "Codebase too long, parsing skipped (%s)",
codebase);
}
{
char *lien = link;
int dejafait = 0;
if (strnotempty(lien) && strlen(lien) < HTS_URLMAXSIZE) {
// calculer les chemins et noms de sauvegarde
if (ident_url_relatif(lien, urladr(), codebase, &afs.af) >= 0) { // reformage selon chemin
int r;
int set_prio_to = 0;
int just_test_it = 0;
forbidden_url =
hts_acceptlink(opt, ptr, afs.af.adr, afs.af.fil, NULL, NULL, &set_prio_to, &just_test_it);
hts_log_print(opt, LOG_DEBUG,
"result for wizard external module link: %d",
forbidden_url);
/* Link accepted */
if (!forbidden_url) {
char BIGSTK tempo[HTS_URLMAXSIZE * 2];
int a, b;
tempo[0] = '\0';
a = opt->savename_type;
b = opt->savename_83;
opt->savename_type = 0;
opt->savename_83 = 0;
// note: adr,fil peuvent être patchés
r =
url_savename(&afs, NULL, NULL, NULL, opt, sback, cache, hashptr, ptr, numero_passe,
NULL);
// resolve unresolved type
if (r != -1 && forbidden_url == 0 && IS_DELAYED_EXT(afs.save)
) { // pas d'erreur, on continue
lien_adrfil former;
former.adr[0] = former.fil[0] = '\0';
r =
hts_wait_delayed(str, &afs, NULL, NULL, &former, &forbidden_url);
}
// end resolve unresolved type
opt->savename_type = a;
opt->savename_83 = b;
if (r != -1 && !forbidden_url) {
if (savename()) {
if (lienrelatif(tempo, afs.save, savename()) == 0) {
hts_log_print(opt, LOG_DEBUG,
"(module): relative link at %s build with %s and %s: %s",
afs.af.adr, afs.save, savename(), tempo);
if (str->localLink
&& str->localLinkSize > (int) strlen(tempo) + 1) {
strcpybuff(str->localLink, tempo);
}
}
}
}
}
if (forbidden_url) {
hts_log_print(opt, LOG_DEBUG, "(module): file not caught: %s",
lien);
if (str->localLink
&& str->localLinkSize > (int) (strlen(afs.af.adr) + strlen(afs.af.fil) + 8)) {
str->localLink[0] = '\0';
if (!link_has_authority(afs.af.adr))
strcpybuff(str->localLink, "http://");
strcatbuff(str->localLink, afs.af.adr);
strcatbuff(str->localLink, afs.af.fil);
}
r = -1;
}
//
if (r != -1) {
hts_log_print(opt, LOG_DEBUG, "(module): %s%s -> %s (base %s)", afs.af.adr,
afs.af.fil, afs.save, codebase);
// modifié par rapport à l'autre version (cf prio_fix notamment et save2)
// vérifier que le lien n'a pas déja été noté
// si c'est le cas, alors il faut s'assurer que la priorité associée
// au fichier est la plus grande des deux priorités
//
// On part de la fin et on essaye de se presser (économise temps machine)
{
int i = hash_read(hashptr, afs.save, NULL, HASH_STRUCT_FILENAME ); // lecture type 0 (sav)
if (i >= 0) {
heap(i)->depth = maximum(heap(i)->depth, prio_fix);
dejafait = 1;
}
}
if (!dejafait) {
//
// >>>> CREER LE LIEN JAVA <<<<
// enregistrer fichier (MACRO)
if (!hts_record_link(opt, afs.af.adr, afs.af.fil, afs.save, "", "", "")) { // erreur, pas de place réservée
opt->state.exit_xh = -1; /* fatal error -> exit */
return 0;
}
// mode test?
heap_top()->testmode = 0; // pas mode test
heap_top()->link_import = 0; // pas mode import
// écrire autres paramètres de la structure-lien
//if (meme_adresse)
heap_top()->premier = heap(ptr)->premier;
//else // sinon l'objet père est le précédent lui même
// heap_top()->premier=ptr;
heap_top()->precedent = ptr;
// noter la priorité
if (!set_prio_to)
heap_top()->depth = prio_fix;
else
heap_top()->depth = max(0, min(heap(ptr)->depth - 1, set_prio_to - 1)); // PRIORITE NULLE (catch page)
heap_top()->pass2 = max(pass_fix, numero_passe);
heap_top()->retry = opt->retry;
//strcpybuff(heap_top()->adr,adr);
//strcpybuff(heap_top()->fil,fil);
//strcpybuff(heap_top()->sav,save);
hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
heap_top()->adr, heap_top()->fil,
heap_top()->sav);
}
}
}
}
}
/* Apply changes */
ENGINE_SAVE_CONTEXT_BASE();
return (forbidden_url == 0);
}
return 0;
}
// message copyright interne
void voidf(void) {
static const char *a;
a = "" CRLF "" CRLF;
a = "+-----------------------------------------------+" CRLF;
a = "|HyperTextTRACKer, Offline Browser Utility |" CRLF;
a = "| HTTrack Website Copier |" CRLF;
a = "|Code: Windows Interface Xavier Roche |" CRLF;
a = "| HTS/HTTrack Xavier Roche |" CRLF;
a = "| .class Parser Yann Philippot |" CRLF;
a = "| |" CRLF;
a = "|Tested on: Windows95,98,NT,2K |" CRLF;
a = "| Linux PC |" CRLF;
a = "| Sun-Solaris 5.6 |" CRLF;
a = "| AIX 4 |" CRLF;
a = "| |" CRLF;
a = "|Copyright (C) Xavier Roche and other |" CRLF;
a = "|contributors |" CRLF;
a = "| |" CRLF;
a = "|Use this program at your own risks! |" CRLF;
a = "+-----------------------------------------------+" CRLF;
a = "" CRLF;
(void) a;
}
// HTTrack Website Copier Copyright (C) 1998-2015 Xavier Roche and other contributors
//
Generated by GNU Enscript 1.6.5.90.