'use strict';
// See index.d.ts for docs
const fetch = require('node-fetch');
const UserAgent = require('user-agents');
exports.snapshot = async function snapshot(args) {
const {
url,
// Without a user agent, force archiving won't work
userAgent = new UserAgent().toString(),
renew = false,
complete = true,
} = args;
// Mutated
let { archiveDomain = 'https://archive.today' } = args;
/** Internal */
let { submitid } = args;
const { referer } = args;
const headers = {};
if (userAgent) headers['user-agent'] = userAgent;
if (referer) headers.referer = referer;
// Get an initial submitid from the index page, as well as follow the redirect for the desired mirror to be used.
// This step is not performed when renewing.
const passedSubmitid = submitid;
if (!submitid) {
const indexPage = await fetch(archiveDomain, { headers });
const body = await indexPage.text();
archiveDomain = indexPage.url;
submitid = extractSubmitid(body);
}
const body =
(passedSubmitid ? 'anyway=1&' : '') +
'url=' +
encodeURIComponent(url) +
'&submitid=' +
encodeURIComponent(submitid);
let response = await fetch(
`${archiveDomain}${archiveDomain.endsWith('/') ? '' : '/'}submit/`,
{
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
...headers,
},
body,
redirect: 'manual',
}
);
let link = '';
let wip = false;
let cachedDate = false;
if (response.status === 200) {
const responseText = await response.text();
if (responseText === '<h1>Invalid URL</h1>') {
throw new TypeError('Invalid URL sent to archive.today: ' + url);
}
link = response.headers.get('refresh').split('url=')[1];
if (link.includes('/wip/')) {
if (complete) {
while (true) {
const res = await fetch(link, {
headers,
redirect: 'manual',
});
const redirect = res.headers.get('location');
if (redirect) {
link = redirect;
break;
}
await new Promise((resolve) => setTimeout(resolve, 1000));
}
} else {
wip = link;
link = link.replace('/wip/', '/');
}
}
} else if (response.status === 302 || response.status === 307) {
// Found
link = response.headers.get('location');
response = await fetch(link, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
...headers,
},
body,
redirect: 'manual',
});
cachedDate = new Date(response.headers.get('memento-datetime'));
// Already archived, re-submit with submitid and referer from the page (this is required)
if (!passedSubmitid) {
const renewing =
typeof renew === 'function' ? await renew(cachedDate) : Boolean(renew);
if (renewing) {
const renewSubmitid = extractSubmitid(await response.text());
return snapshot({
...args,
userAgent, // Re-use the same user agent (if it was generated)
archiveDomain,
submitid: renewSubmitid,
referer: link,
});
}
}
} else {
throw new Error('Unknown response code: ' + response.status);
}
// _ and __ are the // in the protocol
const [_, __, domain, id] = link.split('/');
return { id, url: link, domain, image: `${link}/scr.png`, cachedDate, wip };
};
function extractSubmitid(html) {
return html.match(/<input type="hidden" name="submitid" value="(.*?)"\/>/)[1];
}
exports.timemap = async function timemap({
url,
userAgent = new UserAgent().toString(),
archiveDomain = 'https://archive.today',
}) {
const rawMap = await fetch(`${archiveDomain}/timemap/${url}`, {
headers: userAgent
? {
'User-Agent': userAgent,
}
: {},
}).then((r) => r.text());
const mementoMap = rawMap.split(',\n');
const mementos = [];
// No mementos (never been archived)
if (!mementoMap[0].includes('rel="original"')) {
return mementos;
}
for (const line of mementoMap) {
const [url, rel, datetime] = line.split('; ');
if (rel.includes('memento')) {
// Remove < and >
mementos.push({
url,
date: new Date(datetime.slice(datetime.indexOf('"'), datetime.lastIndexOf('"'))),
});
}
}
return mementos;
};