|
|
@ -4,21 +4,48 @@ require_once './load.php'; |
|
|
require_once './analyse.php'; |
|
|
require_once './analyse.php'; |
|
|
require_once './punycode.php'; |
|
|
require_once './punycode.php'; |
|
|
|
|
|
|
|
|
$html_path = '../html/'; |
|
|
$temp_path = '../temp/'; |
|
|
$release_path = '../release/'; |
|
|
$release_path = '../release/'; |
|
|
|
|
|
|
|
|
|
|
|
shell_exec('mkdir -p ' . $release_path); |
|
|
|
|
|
shell_exec('mkdir -p ' . $temp_path . 'tlds/'); |
|
|
|
|
|
|
|
|
|
|
|
// Get IANA main page |
|
|
|
|
|
echo 'Connect to IANA...'; |
|
|
|
|
|
if (!loadHtmlFile('https://www.iana.org/domains/root/db', $temp_path . 'main.html')) { |
|
|
|
|
|
die('error -> fail to load IANA main page'); |
|
|
|
|
|
} |
|
|
|
|
|
echo "\033[32mOK\033[0m" . PHP_EOL; |
|
|
|
|
|
|
|
|
// Get TLD list from IANA website |
|
|
// Get TLD list from IANA website |
|
|
// main.html -> https://www.iana.org/domains/root/db |
|
|
$html_content = file_get_contents($temp_path . 'main.html'); |
|
|
$tlds = getIanaTlds($html_path . 'main.html'); |
|
|
$tlds = getIanaTlds($html_content); |
|
|
writeFile($release_path . 'tld-list.txt', implode(PHP_EOL, $tlds) . PHP_EOL); |
|
|
writeFile($release_path . 'tld-list.txt', implode(PHP_EOL, $tlds) . PHP_EOL); |
|
|
|
|
|
echo "Found \033[33m" . count($tlds) . "\033[0m TLDs." . PHP_EOL; |
|
|
|
|
|
|
|
|
|
|
|
// Fetch all tld's html file |
|
|
|
|
|
foreach ($tlds as $index => $tld) { |
|
|
|
|
|
$tld = substr($tld, 1 - strlen($tld)); |
|
|
|
|
|
$url = 'https://www.iana.org/domains/root/db/' . $tld . '.html'; |
|
|
|
|
|
echo "\033[36m" . ($index + 1) . '/' . count($tlds) . "\033[0m -> \033[35m." . $tld . "\033[0m"; |
|
|
|
|
|
if (!loadHtmlFile($url, $temp_path . 'tlds/' . $tld . '.html')) { |
|
|
|
|
|
die('error -> fail to load page'); |
|
|
|
|
|
} |
|
|
|
|
|
echo PHP_EOL; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
// Analyse all TLDs from html files |
|
|
// Analyse all TLDs from html files |
|
|
$data = getTldsInfo($tlds, $html_path . 'tlds/'); |
|
|
echo 'Analyse all pages...'; |
|
|
|
|
|
$data = getTldsInfo($tlds, $temp_path . 'tlds/'); |
|
|
|
|
|
echo "\033[32mOK\033[0m" . PHP_EOL; |
|
|
|
|
|
|
|
|
// Output data by json format |
|
|
// Output data by json format |
|
|
|
|
|
echo 'Save as JSON format...'; |
|
|
writeFile($release_path . 'all-data.json', json_encode($data)); |
|
|
writeFile($release_path . 'all-data.json', json_encode($data)); |
|
|
|
|
|
echo "\033[32mOK\033[0m" . PHP_EOL; |
|
|
|
|
|
|
|
|
// Output whois server list by csv format |
|
|
// Output whois server list by csv format |
|
|
|
|
|
echo 'Dump the whois server list...'; |
|
|
$whoisStr = ''; |
|
|
$whoisStr = ''; |
|
|
foreach ($data as $index => $row) { |
|
|
foreach ($data as $index => $row) { |
|
|
if ($row['whois'] !== '') { |
|
|
if ($row['whois'] !== '') { |
|
|
@ -26,8 +53,10 @@ foreach ($data as $index => $row) { |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
writeFile($release_path . 'whois-server.csv', $whoisStr); |
|
|
writeFile($release_path . 'whois-server.csv', $whoisStr); |
|
|
|
|
|
echo "\033[32mOK\033[0m" . PHP_EOL; |
|
|
|
|
|
|
|
|
// Output into sqlite3 database |
|
|
// Output into sqlite3 database |
|
|
|
|
|
echo 'Output into sqlite3 database...'; |
|
|
$init_sql =<<<EOF |
|
|
$init_sql =<<<EOF |
|
|
CREATE TABLE data ( |
|
|
CREATE TABLE data ( |
|
|
tld TEXT NOT NULL, |
|
|
tld TEXT NOT NULL, |
|
|
@ -61,20 +90,10 @@ foreach ($data as $tld => $info) { |
|
|
} |
|
|
} |
|
|
$db->exec($sql . ');'); |
|
|
$db->exec($sql . ');'); |
|
|
} |
|
|
} |
|
|
|
|
|
$db->exec('VACUUM;'); |
|
|
|
|
|
echo "\033[32mOK\033[0m" . PHP_EOL; |
|
|
|
|
|
|
|
|
function writeFile($filename, $data) { |
|
|
// All done |
|
|
$file = fopen($filename, 'w'); |
|
|
echo "\033[32mdone\033[0m" . PHP_EOL; |
|
|
fwrite($file, $data); |
|
|
|
|
|
fclose($file); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
class tldDB extends SQLite3 { |
|
|
|
|
|
public function __construct($filename) { |
|
|
|
|
|
$this->open($filename); |
|
|
|
|
|
} |
|
|
|
|
|
public function __destruct() { |
|
|
|
|
|
$this->close(); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
?> |
|
|
?> |
|
|
|
Dnomd343
3 years ago