small-package/luci-app-homeproxy/root/etc/homeproxy/scripts/update_subscriptions.uc

651 lines
19 KiB
Plaintext
Executable File

#!/usr/bin/ucode
/*
* SPDX-License-Identifier: GPL-2.0-only
*
* Copyright (C) 2023 ImmortalWrt.org
*/
'use strict';
import { open } from 'fs';
import { connect } from 'ubus';
import { cursor } from 'uci';
import { urldecode, urlencode, urldecode_params } from 'luci.http';
import { init_action } from 'luci.sys';
import {
calcStringMD5, wGET, executeCommand, decodeBase64Str,
getTime, isEmpty, parseURL, validation,
HP_DIR, RUN_DIR
} from 'homeproxy';
/* UCI config start */
const uci = cursor();
const uciconfig = 'homeproxy';
uci.load(uciconfig);
const ucimain = 'config',
ucinode = 'node',
ucisubscription = 'subscription';
const allow_insecure = uci.get(uciconfig, ucisubscription, 'allow_insecure') || '0',
filter_mode = uci.get(uciconfig, ucisubscription, 'filter_nodes') || 'disabled',
filter_keywords = uci.get(uciconfig, ucisubscription, 'filter_keywords') || [],
packet_encoding = uci.get(uciconfig, ucisubscription, 'packet_encoding') || 'xudp',
subscription_urls = uci.get(uciconfig, ucisubscription, 'subscription_url') || [],
via_proxy = uci.get(uciconfig, ucisubscription, 'update_via_proxy') || '0';
const routing_mode = uci.get(uciconfig, ucimain, 'routing_mode') || 'bypass_mainalnd_china';
let main_node, main_udp_node;
if (routing_mode !== 'custom') {
main_node = uci.get(uciconfig, ucimain, 'main_node') || 'nil';
main_udp_node = uci.get(uciconfig, ucimain, 'main_udp_node') || 'nil';
}
/* UCI config end */
/* String helper start */
function filter_check(name) {
if (isEmpty(name) || filter_mode === 'disabled' || isEmpty(filter_keywords))
return false;
let ret = false;
for (let i in filter_keywords) {
const patten = regexp(i);
if (match(name, patten))
ret = true;
}
if (filter_mode === 'whitelist')
ret = !ret;
return ret;
}
/* String helper end */
/* Common var start */
const node_cache = {},
node_result = [];
const ubus = connect();
const sing_features = ubus.call('luci.homeproxy', 'singbox_get_features', {}) || {};
/* Common var end */
/* Log */
system(`mkdir -p ${RUN_DIR}`);
function log(...args) {
const logfile = open(`${RUN_DIR}/homeproxy.log`, 'a');
logfile.write(`${getTime()} [SUBSCRIBE] ${join(' ', args)}\n`);
logfile.close();
}
function parse_uri(uri) {
let config, url, params;
if (type(uri) === 'object') {
if (uri.nodetype === 'sip008') {
/* https://shadowsocks.org/guide/sip008.html */
config = {
label: uri.remarks,
type: 'shadowsocks',
address: uri.server,
port: uri.server_port,
shadowsocks_encrypt_method: uri.method,
password: uri.password,
shadowsocks_plugin: uri.plugin,
shadowsocks_plugin_opts: uri.plugin_opts
};
}
} else if (type(uri) === 'string') {
uri = split(trim(uri), '://');
switch (uri[0]) {
case 'http':
case 'https':
url = parseURL('http://' + uri[1]);
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'http',
address: url.hostname,
port: url.port,
username: url.username ? urldecode(url.username) : null,
password: url.password ? urldecode(url.password) : null,
tls: (uri[0] === 'https') ? '1' : '0'
};
break;
case 'hysteria':
/* https://github.com/HyNetwork/hysteria/wiki/URI-Scheme */
url = parseURL('http://' + uri[1]);
params = url.searchParams;
if (!sing_features.with_quic || (params.protocol && params.protocol !== 'udp')) {
log(sprintf('Skipping unsupported %s node: %s.', 'hysteria', urldecode(url.hash) || url.hostname));
if (!sing_features.with_quic)
log(sprintf('Please rebuild sing-box with %s support!', 'QUIC'));
return null;
}
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'hysteria',
address: url.hostname,
port: url.port,
hysteria_protocol: params.protocol || 'udp',
hysteria_auth_type: params.auth ? 'string' : null,
hysteria_auth_payload: params.auth,
hysteria_obfs_password: params.obfsParam,
hysteria_down_mbps: params.downmbps,
hysteria_up_mbps: params.upmbps,
tls: '1',
tls_insecure: (params.insecure in ['true', '1']) ? '1' : '0',
tls_sni: params.peer,
tls_alpn: params.alpn
};
break;
case 'hysteria2':
case 'hy2':
/* https://v2.hysteria.network/docs/developers/URI-Scheme/ */
url = parseURL('http://' + uri[1]);
params = url.searchParams;
/* userpass auth is not supported by sing-box */
if (!sing_features.with_quic || (params.password)) {
log(sprintf('Skipping unsupported %s node: %s.', 'hysteria2', urldecode(url.hash) || url.hostname));
if (!sing_features.with_quic)
log(sprintf('Please rebuild sing-box with %s support!', 'QUIC'));
return null;
}
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'hysteria2',
address: url.hostname,
port: url.port,
password: url.password ? urldecode(url.password) : null,
hysteria_obfs_type: params.obfs,
hysteria_obfs_password: params['obfs-password'],
tls: '1',
tls_insecure: params.insecure ? '1' : '0',
tls_sni: params.sni
};
break;
case 'socks':
case 'socks4':
case 'socks4a':
case 'socsk5':
case 'socks5h':
url = parseURL('http://' + uri[1]);
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'socks',
address: url.hostname,
port: url.port,
username: url.username ? urldecode(url.username) : null,
password: url.password ? urldecode(url.password) : null,
socks_version: (match(uri[0], /4/)) ? '4' : '5'
};
break;
case 'ss':
/* "Lovely" Shadowrocket format */
const ss_suri = split(uri[1], '#');
let ss_slabel = '';
if (length(ss_suri) <= 2) {
if (length(ss_suri) === 2)
ss_slabel = '#' + urlencode(ss_suri[1]);
if (decodeBase64Str(ss_suri[0]))
uri[1] = decodeBase64Str(ss_suri[0]) + ss_slabel;
}
/* Legacy format is not supported, it should be never appeared in modern subscriptions */
/* https://github.com/shadowsocks/shadowsocks-org/commit/78ca46cd6859a4e9475953ed34a2d301454f579e */
/* SIP002 format https://shadowsocks.org/guide/sip002.html */
url = parseURL('http://' + uri[1]);
let ss_userinfo = {};
if (url.username && url.password)
/* User info encoded with URIComponent */
ss_userinfo = [url.username, urldecode(url.password)];
else if (url.username)
/* User info encoded with base64 */
ss_userinfo = split(decodeBase64Str(urldecode(url.username)), ':');
let ss_plugin, ss_plugin_opts;
if (url.search && url.searchParams.plugin) {
const ss_plugin_info = split(url.searchParams.plugin, ';');
ss_plugin = ss_plugin_info[0];
if (ss_plugin === 'simple-obfs')
/* Fix non-standard plugin name */
ss_plugin = 'obfs-local';
ss_plugin_opts = slice(ss_plugin_info, 1) ? join(';', slice(ss_plugin_info, 1)) : null;
}
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'shadowsocks',
address: url.hostname,
port: url.port,
shadowsocks_encrypt_method: ss_userinfo[0],
password: ss_userinfo[1],
shadowsocks_plugin: ss_plugin,
shadowsocks_plugin_opts: ss_plugin_opts
};
break;
case 'ssr':
/* https://coderschool.cn/2498.html */
uri = split(decodeBase64Str(uri[1]), '/');
if (!uri)
return null;
const userinfo = split(uri[0], ':');
params = urldecode_params(uri[1]);
if (!sing_features.with_shadowsocksr) {
log(sprintf('Skipping unsupported %s node: %s.', 'ShadowsocksR', decodeBase64Str(params.remarks) || userinfo[1]));
log(sprintf('Please rebuild sing-box with %s support!', 'ShadowsocksR'));
return null;
}
config = {
label: decodeBase64Str(params.remarks),
type: 'shadowsocksr',
address: userinfo[0],
port: userinfo[1],
shadowsocksr_encrypt_method: userinfo[3],
password: decodeBase64Str(userinfo[5]),
shadowsocksr_protocol: userinfo[2],
shadowsocksr_protocol_param: decodeBase64Str(params.protoparam),
shadowsocksr_obfs: userinfo[4],
shadowsocksr_obfs_param: decodeBase64Str(params.obfsparam)
};
break;
case 'trojan':
/* https://p4gefau1t.github.io/trojan-go/developer/url/ */
url = parseURL('http://' + uri[1]);
params = url.searchParams || {};
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'trojan',
address: url.hostname,
port: url.port,
password: urldecode(url.username),
transport: (params.type !== 'tcp') ? params.type : null,
tls: '1',
tls_sni: params.sni
};
switch(params.type) {
case 'grpc':
config.grpc_servicename = params.serviceName;
break;
case 'ws':
/* We don't parse "host" param when TLS is enabled, as some providers are abusing it (host vs sni)
* config.ws_host = params.host ? urldecode(params.host) : null;
*/
config.ws_path = params.path ? urldecode(params.path) : null;
if (config.ws_path && match(config.ws_path, /\?ed=/)) {
config.websocket_early_data_header = 'Sec-WebSocket-Protocol';
config.websocket_early_data = split(config.ws_path, '?ed=')[1];
config.ws_path = split(config.ws_path, '?ed=')[0];
}
break;
}
break;
case 'tuic':
/* https://github.com/daeuniverse/dae/discussions/182 */
url = parseURL('http://' + uri[1]);
params = url.searchParams || {};
if (!sing_features.with_quic) {
log(sprintf('Skipping unsupported %s node: %s.', 'tuic', urldecode(url.hash) || url.hostname));
log(sprintf('Please rebuild sing-box with %s support!', 'QUIC'));
return null;
}
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'tuic',
address: url.hostname,
port: url.port,
uuid: url.username,
password: url.password ? urldecode(url.password) : null,
tuic_congestion_control: params.congestion_control,
tuic_udp_relay_mode: params.udp_relay_mode,
tls: '1',
tls_sni: params.sni,
tls_alpn: params.alpn ? split(urldecode(params.alpn), ',') : null,
};
break;
case 'vless':
/* https://github.com/XTLS/Xray-core/discussions/716 */
url = parseURL('http://' + uri[1]);
params = url.searchParams;
/* Unsupported protocol */
if (params.type === 'kcp') {
log(sprintf('Skipping sunsupported %s node: %s.', 'VLESS', urldecode(url.hash) || url.hostname));
return null;
} else if (params.type === 'quic' && (params.quicSecurity && params.quicSecurity !== 'none' || !sing_features.with_quic)) {
log(sprintf('Skipping sunsupported %s node: %s.', 'VLESS', urldecode(url.hash) || url.hostname));
if (!sing_features.with_quic)
log(sprintf('Please rebuild sing-box with %s support!', 'QUIC'));
return null;
}
config = {
label: url.hash ? urldecode(url.hash) : null,
type: 'vless',
address: url.hostname,
port: url.port,
uuid: url.username,
transport: (params.type !== 'tcp') ? params.type : null,
tls: (params.security in ['tls', 'xtls', 'reality']) ? '1' : '0',
tls_sni: params.sni,
tls_alpn: params.alpn ? split(urldecode(params.alpn), ',') : null,
tls_reality: (params.security === 'reality') ? '1' : '0',
tls_reality_public_key: params.pbk ? urldecode(params.pbk) : null,
tls_reality_short_id: params.sid,
tls_utls: sing_features.with_utls ? params.fp : null,
vless_flow: (params.security in ['tls', 'reality']) ? params.flow : null
};
switch(params.type) {
case 'grpc':
config.grpc_servicename = params.serviceName;
break;
case 'http':
case 'tcp':
if (config.transport === 'http' || params.headerType === 'http') {
config.http_host = params.host ? split(urldecode(params.host), ',') : null;
config.http_path = params.path ? urldecode(params.path) : null;
}
break;
case 'ws':
/* We don't parse "host" param when TLS is enabled, as some providers are abusing it (host vs sni) */
config.ws_host = (config.tls !== '1' && params.host) ? urldecode(params.host) : null;
config.ws_path = params.path ? urldecode(params.path) : null;
if (config.ws_path && match(config.ws_path, /\?ed=/)) {
config.websocket_early_data_header = 'Sec-WebSocket-Protocol';
config.websocket_early_data = split(config.ws_path, '?ed=')[1];
config.ws_path = split(config.ws_path, '?ed=')[0];
}
break;
}
break;
case 'vmess':
/* "Lovely" shadowrocket format */
if (match(uri, /&/)) {
log(sprintf('Skipping unsupported %s format.', 'VMess'));
return null;
}
/* https://github.com/2dust/v2rayN/wiki/%E5%88%86%E4%BA%AB%E9%93%BE%E6%8E%A5%E6%A0%BC%E5%BC%8F%E8%AF%B4%E6%98%8E(ver-2) */
try {
uri = json(decodeBase64Str(uri[1]));
} catch(e) {
log(sprintf('Skipping unsupported %s format.', 'VMess'));
return null;
}
if (uri.v !== '2') {
log(sprintf('Skipping unsupported %s format.', 'VMess'));
return null;
/* Unsupported protocol */
} else if (uri.net === 'kcp') {
log(sprintf('Skipping unsupported %s node: %s.', 'VMess', uri.ps || uri.add));
return null;
} else if (uri.net === 'quic' && ((uri.type && uri.type !== 'none') || uri.path || !sing_features.with_quic)) {
log(sprintf('Skipping unsupported %s node: %s.', 'VMess', uri.ps || uri.add));
if (!sing_features.with_quic)
log(sprintf('Please rebuild sing-box with %s support!', 'QUIC'));
return null;
}
/*
* https://www.v2fly.org/config/protocols/vmess.html#vmess-md5-%E8%AE%A4%E8%AF%81%E4%BF%A1%E6%81%AF-%E6%B7%98%E6%B1%B0%E6%9C%BA%E5%88%B6
* else if (uri.aid && int(uri.aid) !== 0) {
* log(sprintf('Skipping unsupported %s node: %s.', 'VMess', uri.ps || uri.add));
* return null;
* }
*/
config = {
label: uri.ps,
type: 'vmess',
address: uri.add,
port: uri.port,
uuid: uri.id,
vmess_alterid: uri.aid,
vmess_encrypt: uri.scy || 'auto',
vmess_global_padding: '1',
transport: (uri.net !== 'tcp') ? uri.net : null,
tls: (uri.tls === 'tls') ? '1' : '0',
tls_sni: uri.sni || uri.host,
tls_alpn: uri.alpn ? split(uri.alpn, ',') : null
};
switch (uri.net) {
case 'grpc':
config.grpc_servicename = uri.path;
break;
case 'h2':
case 'tcp':
if (uri.net === 'h2' || uri.type === 'http') {
config.transport = 'http';
config.http_host = uri.host ? uri.host.split(',') : null;
config.http_path = uri.path;
}
break;
case 'ws':
/* We don't parse "host" param when TLS is enabled, as some providers are abusing it (host vs sni) */
config.ws_host = (config.tls !== '1') ? uri.host : null;
config.ws_path = uri.path;
if (config.ws_path && match(config.ws_path, /\?ed=/)) {
config.websocket_early_data_header = 'Sec-WebSocket-Protocol';
config.websocket_early_data = split(config.ws_path, '?ed=')[1];
config.ws_path = split(config.ws_path, '?ed=')[0];
}
break;
}
break;
}
}
if (!isEmpty(config)) {
if (config.address)
config.address = replace(config.address, /\[|\]/g, '');
if (!validation('host', config.address) || !validation('port', config.port)) {
log(sprintf('Skipping invalid %s node: %s.', config.type, config.label || 'NULL'));
return null;
} else if (!config.label)
config.label = (validation('ip6addr', config.address) ?
`[${config.address}]` : config.address) + ':' + config.port;
}
return config;
}
function main() {
if (via_proxy !== '1') {
log('Stopping service...');
init_action('homeproxy', 'stop');
}
for (let url in subscription_urls) {
const groupHash = calcStringMD5(url);
node_cache[groupHash] = {};
const res = wGET(url);
if (!res) {
log(sprintf('Failed to fetch resources from %s.', url));
continue;
}
push(node_result, []);
const subindex = length(node_result) - 1;
let nodes;
try {
nodes = json(res).servers || json(res);
/* Shadowsocks SIP008 format */
if (nodes[0].server && nodes[0].method)
map(nodes, (_, i) => nodes[i].nodetype = 'sip008');
} catch(e) {
nodes = decodeBase64Str(res);
nodes = nodes ? split(trim(replace(nodes, / /g, '_')), '\n') : {};
}
let count = 0;
for (let node in nodes) {
let config;
if (!isEmpty(node))
config = parse_uri(node);
if (isEmpty(config))
continue;
const label = config.label;
config.label = null;
const confHash = calcStringMD5(sprintf('%J', config)),
nameHash = calcStringMD5(label);
config.label = label;
if (filter_check(config.label))
log(sprintf('Skipping blacklist node: %s.', config.label));
else if (node_cache[groupHash][confHash] || node_cache[groupHash][nameHash])
log(sprintf('Skipping duplicate node: %s.', config.label));
else {
if (config.tls === '1' && allow_insecure === '1')
config.tls_insecure = '1';
if (config.type in ['vless', 'vmess'])
config.packet_encoding = packet_encoding;
config.grouphash = groupHash;
push(node_result[subindex], config);
node_cache[groupHash][confHash] = config;
node_cache[groupHash][nameHash] = config;
count++;
}
}
log(sprintf('Successfully fetched %s nodes of total %s from %s.', count, length(nodes), url));
}
if (isEmpty(node_result)) {
log('Failed to update subscriptions: no valid node found.');
if (via_proxy !== '1') {
log('Starting service...');
init_action('homeproxy', 'start');
}
return false;
}
let added = 0, removed = 0;
uci.foreach(uciconfig, ucinode, (cfg) => {
/* Nodes created by the user */
if (!cfg.grouphash)
return null;
/* Empty object - failed to fetch nodes */
if (length(node_cache[cfg.grouphash]) === 0)
return null;
if (!node_cache[cfg.grouphash] || !node_cache[cfg.grouphash][cfg['.name']]) {
uci.delete(uciconfig, cfg['.name']);
removed++;
log(sprintf('Removing node: %s.', cfg.label || cfg['name']));
} else {
map(keys(node_cache[cfg.grouphash][cfg['.name']]), (v) => {
uci.set(uciconfig, cfg['.name'], v, node_cache[cfg.grouphash][cfg['.name']][v]);
});
node_cache[cfg.grouphash][cfg['.name']].isExisting = true;
}
});
for (let nodes in node_result)
map(nodes, (node) => {
if (node.isExisting)
return null;
const nameHash = calcStringMD5(node.label);
uci.set(uciconfig, nameHash, 'node');
map(keys(node), (v) => uci.set(uciconfig, nameHash, v, node[v]));
added++;
log(sprintf('Adding node: %s.', node.label));
});
uci.commit(uciconfig);
let need_restart = (via_proxy !== '1');
if (!isEmpty(main_node)) {
const first_server = uci.get_first(uciconfig, ucinode);
if (first_server) {
if (!uci.get(uciconfig, main_node)) {
uci.set(uciconfig, ucimain, 'main_node', first_server);
uci.commit(uciconfig);
need_restart = true;
log('Main node is gone, switching to the first node.');
}
if (!isEmpty(main_udp_node) && main_udp_node !== 'same') {
if (!uci.get(uciconfig, main_udp_node)) {
uci.set(uciconfig, ucimain, 'main_udp_node', first_server);
uci.commit(uciconfig);
need_restart = true;
log('Main UDP node is gone, switching to the first node.');
}
}
} else {
uci.set(uciconfig, ucimain, 'main_node', 'nil');
uci.set(uciconfig, ucimain, 'main_udp_node', 'nil');
uci.commit(uciconfig);
need_restart = true;
log('No available node, disable tproxy.');
}
}
if (need_restart) {
log('Restarting service...');
init_action('homeproxy', 'stop');
init_action('homeproxy', 'start');
}
log(sprintf('%s nodes added, %s removed.', added, removed));
log('Successfully updated subscriptions.');
}
if (!isEmpty(subscription_urls))
try {
call(main);
} catch(e) {
log('[FATAL ERROR] An error occurred during updating subscriptions:');
log(sprintf('%s: %s', e.type, e.message));
log(e.stacktrace[0].context);
log('Restarting service...');
init_action('homeproxy', 'stop');
init_action('homeproxy', 'start');
}