From 41dec5046a81c54d58d8eda9ca8bd259455cadc0 Mon Sep 17 00:00:00 2001 From: gorhill Date: Sat, 14 Jan 2017 17:46:15 -0500 Subject: [PATCH 01/15] refactoring assets management code --- assets/assets.json | 569 +++++++++ src/3p-filters.html | 13 +- src/_locales/en/messages.json | 4 +- src/background.html | 1 - src/css/3p-filters.css | 76 +- src/js/3p-filters.js | 254 ++-- src/js/assets.js | 1988 +++++++++---------------------- src/js/background.js | 89 +- src/js/logger.js | 4 +- src/js/messaging.js | 31 +- src/js/redirect-engine.js | 24 +- src/js/reverselookup.js | 30 +- src/js/scriptlets/subscriber.js | 9 +- src/js/start.js | 25 +- src/js/storage.js | 848 +++++-------- src/js/traffic.js | 2 +- src/js/ublock.js | 8 +- tools/make-assets.sh | 4 +- tools/make-chromium.sh | 6 +- 19 files changed, 1647 insertions(+), 2338 deletions(-) create mode 100644 assets/assets.json diff --git a/assets/assets.json b/assets/assets.json new file mode 100644 index 0000000000000..8a165b43ef90e --- /dev/null +++ b/assets/assets.json @@ -0,0 +1,569 @@ +{ + "assets.json": { + "content": "internal", + "updateAfter": 13, + "contentURL": [ + "https://raw.githubusercontent.com/gorhill/uBlock/master/assets/assets.json", + "assets/assets.json" + ] + }, + "public_suffix_list.dat": { + "content": "internal", + "updateAfter": 19, + "contentURL": [ + "https://publicsuffix.org/list/public_suffix_list.dat", + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat" + ] + }, + "ublock-resources": { + "content": "internal", + "updateAfter": 7, + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resources.txt", + "assets/ublock/resources.txt" + ] + }, + "ublock-filters": { + "content": "filters", + "title": "uBlock filters", + "group": "default", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt", + "assets/ublock/filters.txt" + ] + }, + "ublock-privacy": { + "content": "filters", + "title": "uBlock filters – Privacy", + "group": "default", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt", + "assets/ublock/privacy.txt" + ] + }, + "ublock-unbreak": { + "content": "filters", + "title": "uBlock filters – Unbreak", + "group": "default", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt", + "assets/ublock/unbreak.txt" + ] + }, + "ublock-badware": { + "content": "filters", + "title": "uBlock filters – Badware risks", + "group": "default", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt", + "assets/ublock/badware.txt" + ], + "supportURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks", + "instructionURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks" + }, + "ublock-experimental": { + "content": "filters", + "title": "uBlock filters – Experimental", + "group": "default", + "off": true, + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/experimental.txt", + "assets/ublock/experimental.txt" + ], + "supportURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters", + "instructionURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters" + }, + "CHN-0": { + "content": "filters", + "off": true, + "title": "CHN: EasyList China (中文)", + "group": "regions", + "lang": "zh", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistchina.txt", + "supportURL": "http://abpchina.org/forum/forum.php" + }, + "CHN-1": { + "content": "filters", + "off": true, + "title": "CHN: CJX's EasyList Lite (main focus on Chinese sites)", + "group": "regions", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" + }, + "CHN-2": { + "content": "filters", + "off": true, + "title": "CHN: CJX's Annoyance List", + "group": "regions", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" + }, + "DEU-0": { + "content": "filters", + "off": true, + "title": "DEU: EasyList Germany", + "group": "regions", + "lang": "de", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=90" + }, + "DNK-0": { + "content": "filters", + "off": true, + "title": "DNK: Schacks Adblock Plus liste", + "group": "regions", + "lang": "da", + "contentURL": "https://adblock.dk/block.csv", + "supportURL": "https://henrik.schack.dk/adblock/" + }, + "easylist": { + "content": "filters", + "title": "EasyList", + "group": "ads", + "contentURL": [ + "https://easylist.to/easylist/easylist.txt", + "https://easylist-downloads.adblockplus.org/easylist.txt", + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easylist.txt", + "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "easylist-nocosmetic": { + "content": "filters", + "off": true, + "title": "EasyList without element hiding rules", + "group": "ads", + "contentURL": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt", + "supportURL": "https://forums.lanik.us/" + }, + "easyprivacy": { + "content": "filters", + "title": "EasyPrivacy", + "group": "privacy", + "contentURL": [ + "https://easylist.to/easylist/easyprivacy.txt", + "https://easylist-downloads.adblockplus.org/easyprivacy.txt", + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt", + "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-annoyance": { + "content": "filters", + "off": true, + "title": "Fanboy’s Annoyance List", + "group": "social", + "contentURL": [ + "https://easylist.to/easylist/fanboy-annoyance.txt", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-social": { + "content": "filters", + "off": true, + "title": "Fanboy’s Social Blocking List", + "group": "social", + "contentURL": [ + "https://easylist.to/easylist/fanboy-social.txt", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "FRA-0": { + "content": "filters", + "off": true, + "title": "FRA: EasyList Liste FR", + "group": "regions", + "lang": "fr", + "contentURL": "https://easylist-downloads.adblockplus.org/liste_fr.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=91" + }, + "ISL-0": { + "content": "filters", + "off": true, + "title": "ISL: Icelandic ABP List", + "group": "regions", + "lang": "is", + "contentURL": "http://adblock.gardar.net/is.abp.txt", + "supportURL": "http://adblock.gardar.net/" + }, + "ITA-0": { + "content": "filters", + "off": true, + "title": "ITA: EasyList Italy", + "group": "regions", + "lang": "it", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=96" + }, + "ITA-1": { + "content": "filters", + "off": true, + "title": "ITA: ABP X Files", + "group": "regions", + "contentURL": "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt", + "supportURL": "http://noads.it/" + }, + "RUS-0": { + "content": "filters", + "off": true, + "title": "RUS: RU AdList (Дополнительная региональная подписка)", + "group": "regions", + "lang": "ru", + "contentURL": "https://easylist-downloads.adblockplus.org/advblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + }, + "RUS-1": { + "content": "filters", + "off": true, + "title": "RUS: BitBlock List (Дополнительная подписка фильтров)", + "group": "regions", + "contentURL": "https://easylist-downloads.adblockplus.org/bitblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + }, + "RUS-2": { + "content": "filters", + "off": true, + "title": "RUS: Adguard Russian Filter", + "group": "regions", + "contentURL": "https://filters.adtidy.org/extension/chromium/filters/1.txt", + "supportURL": "https://forum.adguard.com/forumdisplay.php?69-%D0%A4%D0%B8%D0%BB%D1%8C%D1%82%D1%80%D1%8B-Adguard" + }, + "NLD-0": { + "content": "filters", + "off": true, + "title": "NLD: EasyList Dutch", + "group": "regions", + "lang": "nl", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=100" + }, + "LVA-0": { + "content": "filters", + "off": true, + "title": "LVA: Latvian List", + "group": "regions", + "lang": "lv", + "contentURL": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", + "supportURL": "https://notabug.org/latvian-list/adblock-latvian" + }, + "hphosts": { + "content": "filters", + "off": true, + "title": "hpHosts’ Ad and tracking servers", + "group": "multipurpose", + "contentURL": "http://hosts-file.net/.%5Cad_servers.txt", + "supportURL": "http://hosts-file.net/" + }, + "EST-0": { + "content": "filters", + "off": true, + "title": "EST: Eesti saitidele kohandatud filter", + "group": "regions", + "lang": "et", + "contentURL": "http://adblock.ee/list.php", + "supportURL": "http://adblock.ee/" + }, + "disconnect-malvertising": { + "content": "filters", + "off": true, + "title": "Malvertising filter list by Disconnect", + "group": "malware", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt" + }, + "disconnect-malware": { + "content": "filters", + "off": true, + "title": "Malware filter list by Disconnect", + "group": "malware", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt" + }, + "disconnect-tracking": { + "content": "filters", + "off": true, + "title": "Basic tracking list by Disconnect", + "group": "privacy", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" + }, + "POL-0": { + "content": "filters", + "off": true, + "title": "POL: polskie filtry do Adblocka i uBlocka", + "group": "regions", + "lang": "pl", + "contentURL": "https://www.certyficate.it/adblock/adblock.txt", + "supportURL": "http://www.certyficate.it/adblock-ublock-polish-filters/" + }, + "awrl-0": { + "content": "filters", + "off": true, + "title": "Adblock Warning Removal List", + "group": "ads", + "contentURL": "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt", + "supportURL": "https://forums.lanik.us/" + }, + "FIN-0": { + "content": "filters", + "off": true, + "title": "FIN: Finnish Addition to Easylist", + "group": "regions", + "lang": "fi", + "contentURL": "http://adb.juvander.net/Finland_adb.txt", + "supportURL": "http://www.juvander.fi/AdblockFinland" + }, + "KOR-0": { + "content": "filters", + "off": true, + "title": "KOR: Korean Adblock List", + "group": "regions", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt", + "supportURL": "https://github.com/gfmaster/adblock-korea-contrib" + }, + "KOR-1": { + "content": "filters", + "off": true, + "title": "KOR: YousList", + "group": "regions", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt", + "supportURL": "https://github.com/yous/YousList" + }, + "KOR-2": { + "content": "filters", + "off": true, + "title": "KOR: Fanboy's Korean", + "group": "regions", + "contentURL": "https://www.fanboy.co.nz/fanboy-korean.txt", + "supportURL": "https://forums.lanik.us/" + }, + "IDN-0": { + "content": "filters", + "off": true, + "title": "IDN: ABPindo", + "group": "regions", + "lang": "id", + "contentURL": "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt", + "supportURL": "https://github.com/heradhis/indonesianadblockrules" + }, + "JPN-0": { + "content": "filters", + "off": true, + "title": "JPN: ABP Japanese filters (日本用フィルタ)", + "group": "regions", + "lang": "ja", + "contentURL": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", + "supportURL": "https://github.com/k2jp/abp-japanese-filters/wiki/Support_Policy" + }, + "EU-prebake": { + "content": "filters", + "off": true, + "title": "EU: Prebake - Filter Obtrusive Cookie Notices", + "group": "regions", + "contentURL": "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt", + "supportURL": "https://github.com/liamja/Prebake" + }, + "ara-0": { + "content": "filters", + "off": true, + "title": "ara: Liste AR", + "group": "regions", + "lang": "ar", + "contentURL": "https://easylist-downloads.adblockplus.org/Liste_AR.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=98" + }, + "LTU-0": { + "content": "filters", + "off": true, + "title": "LTU: Adblock Plus Lithuania", + "group": "regions", + "lang": "lt", + "contentURL": "http://margevicius.lt/easylistlithuania.txt", + "supportURL": "http://margevicius.lt/easylist_lithuania/" + }, + "malware-0": { + "content": "filters", + "off": true, + "title": "Malware domains (long-lived)", + "group": "malware", + "contentURL": "http://malwaredomains.lehigh.edu/files/immortal_domains.txt", + "supportURL": "http://www.malwaredomains.com/" + }, + "malware-1": { + "content": "filters", + "title": "Malware Domain List", + "group": "malware", + "contentURL": [ + "https://www.malwaredomainlist.com/hostslist/hosts.txt", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt" + ] + }, + "malware-2": { + "content": "filters", + "title": "Malware domains", + "group": "malware", + "contentURL": [ + "https://mirror.cedia.org.ec/malwaredomains/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains" + ], + "supportURL": "http://www.malwaredomains.com/" + }, + "plowe-0": { + "content": "filters", + "title": "Peter Lowe’s Ad and tracking server list", + "group": "multipurpose", + "contentURL": [ + "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext", + "assets/thirdparties/pgl.yoyo.org/as/serverlist" + ], + "supportURL": "https://pgl.yoyo.org/adservers/" + }, + "ISR-0": { + "content": "filters", + "off": true, + "title": "ISR: EasyList Hebrew", + "group": "regions", + "lang": "he", + "contentURL": "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt", + "supportURL": "https://github.com/easylist/EasyListHebrew" + }, + "reek-0": { + "content": "filters", + "off": true, + "title": "Anti-Adblock Killer | Reek", + "group": "ads", + "contentURL": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", + "supportURL": "https://github.com/reek/anti-adblock-killer", + "instructionURL": "https://github.com/reek/anti-adblock-killer#instruction" + }, + "HUN-0": { + "content": "filters", + "off": true, + "title": "HUN: hufilter", + "group": "regions", + "lang": "hu", + "contentURL": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", + "supportURL": "https://github.com/szpeter80/hufilter" + }, + "CZE-0": { + "content": "filters", + "off": true, + "title": "CZE, SVK: EasyList Czech and Slovak", + "group": "regions", + "lang": "cs", + "contentURL": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", + "supportURL": "https://github.com/tomasko126/easylistczechandslovak" + }, + "dpollock-0": { + "content": "filters", + "off": true, + "title": "Dan Pollock’s hosts file", + "group": "multipurpose", + "contentURL": "http://someonewhocares.org/hosts/hosts", + "supportURL": "http://someonewhocares.org/hosts/" + }, + "spam404-0": { + "content": "filters", + "off": true, + "title": "Spam404", + "group": "malware", + "contentURL": "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt", + "supportURL": "http://www.spam404.com/" + }, + "BGR-0": { + "content": "filters", + "off": true, + "title": "BGR: Bulgarian Adblock list", + "group": "regions", + "lang": "bg", + "contentURL": "http://stanev.org/abp/adblock_bg.txt", + "supportURL": "http://stanev.org/abp/" + }, + "mvps-0": { + "content": "filters", + "off": true, + "title": "MVPS HOSTS", + "group": "multipurpose", + "contentURL": "http://winhelp2002.mvps.org/hosts.txt", + "supportURL": "http://winhelp2002.mvps.org/" + }, + "fanboy-enhanced": { + "content": "filters", + "off": true, + "title": "Fanboy’s Enhanced Tracking List", + "group": "privacy", + "contentURL": "https://www.fanboy.co.nz/enhancedstats.txt", + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-thirdparty_social": { + "content": "filters", + "off": true, + "title": "Anti-ThirdpartySocial (see warning inside list)", + "group": "social", + "contentURL": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", + "supportURL": "https://forums.lanik.us/" + }, + "spa-0": { + "content": "filters", + "off": true, + "title": "spa: EasyList Spanish", + "group": "regions", + "lang": "es", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=103" + }, + "SWE-0": { + "content": "filters", + "off": true, + "title": "SWE: Fanboy's Swedish", + "group": "regions", + "lang": "sv", + "contentURL": "https://www.fanboy.co.nz/fanboy-swedish.txt", + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-ultimate": { + "content": "filters", + "off": true, + "title": "Fanboy+Easylist-Merged Ultimate List", + "group": "multipurpose", + "contentURL": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", + "supportURL": "https://forums.lanik.us/" + }, + "TUR-0": { + "content": "filters", + "off": true, + "title": "TUR: Adguard Turkish Filter", + "group": "regions", + "lang": "tr", + "contentURL": "https://filters.adtidy.org/extension/chromium/filters/13.txt", + "supportURL": "https://forum.adguard.com/forumdisplay.php?51-Filter-Rules" + }, + "VIE-0": { + "content": "filters", + "off": true, + "title": "VIE: Fanboy's Vietnamese", + "group": "regions", + "lang": "vi", + "contentURL": "https://www.fanboy.co.nz/fanboy-vietnam.txt", + "supportURL": "https://forums.lanik.us/" + }, + "GRC-0": { + "content": "filters", + "off": true, + "title": "GRC: Greek AdBlock Filter", + "group": "regions", + "lang": "el", + "contentURL": "https://www.void.gr/kargig/void-gr-filters.txt", + "supportURL": "https://github.com/kargig/greek-adblockplus-filter" + }, + "SVN-0": { + "content": "filters", + "off": true, + "title": "SVN: Slovenian List", + "group": "regions", + "lang": "sl", + "contentURL": "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt", + "supportURL": "https://github.com/betterwebleon/slovenian-list" + } +} diff --git a/src/3p-filters.html b/src/3p-filters.html index 8bce9d01aaab4..faca189972f08 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -40,12 +40,6 @@

-
-
- -
-
- diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json index b31059c90154c..653cf75c40c94 100644 --- a/src/_locales/en/messages.json +++ b/src/_locales/en/messages.json @@ -580,8 +580,8 @@ "description": "Message asking user to confirm reset" }, "errorCantConnectTo":{ - "message":"Unable to connect to {{url}}", - "description":"English: Network error: unable to connect to {{url}}" + "message":"Network error: {{msg}}", + "description":"English: Network error: {{msg}}" }, "subscriberConfirm":{ "message":"uBlock₀: Add the following URL to your custom filter lists?\n\nTitle: \"{{title}}\"\nURL: {{url}}", diff --git a/src/background.html b/src/background.html index a8463068e76c3..0d56942fc2b0e 100644 --- a/src/background.html +++ b/src/background.html @@ -8,7 +8,6 @@ - diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index 99f87cfd48235..207318bff8240 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -98,19 +98,30 @@ span.status { span.unsecure { background-color: hsl(0, 100%, 88%); border-color: hsl(0, 100%, 83%); + display: none; + } +li.listEntry.unsecure span.unsecure { + display: unset; } span.purge { border-color: #ddd; background-color: #eee; cursor: pointer; + display: none; } span.purge:hover { opacity: 1; } -span.obsolete, -span.new { +li.listEntry.cached span.purge { + display: unset; + } +span.obsolete { background-color: hsl(36, 100%, 80%); border-color: hsl(36, 100%, 75%); + display: none; + } +li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete { + display: unset; } #externalListsDiv { margin: 2em auto 0 2em; @@ -125,64 +136,3 @@ body[dir=rtl] #externalListsDiv { width: 100%; word-wrap: normal; } -body #busyOverlay { - background-color: transparent; - bottom: 0; - cursor: wait; - display: none; - left: 0; - position: fixed; - right: 0; - top: 0; - z-index: 1000; - } -body.busy #busyOverlay { - display: block; - } -#busyOverlay > div:nth-of-type(1) { - background-color: white; - bottom: 0; - left: 0; - opacity: 0.75; - position: absolute; - right: 0; - top: 0; - } -#busyOverlay > div:nth-of-type(2) { - background-color: #eee; - border: 1px solid transparent; - border-color: #80b3ff #80b3ff hsl(216, 100%, 75%); - border-radius: 3px; - box-sizing: border-box; - height: 3em; - left: 10%; - position: absolute; - bottom: 75%; - width: 80%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(1) { - background-color: hsl(216, 100%, 75%); - background-image: linear-gradient(#a8cbff, #80b3ff); - background-repeat: repeat-x; - border: 0; - box-sizing: border-box; - color: #222; - height: 100%; - left: 0; - padding: 0; - position: absolute; - width: 25%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(2) { - background-color: transparent; - border: 0; - box-sizing: border-box; - height: 100%; - left: 0; - line-height: 3em; - overflow: hidden; - position: absolute; - text-align: center; - top: 0; - width: 100%; - } diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 0b960e81c0911..fc2d2509b4418 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,38 +21,29 @@ /* global uDom */ +'use strict'; + /******************************************************************************/ (function() { -'use strict'; - /******************************************************************************/ -var userListName = vAPI.i18n('1pPageName'); var listDetails = {}; var parseCosmeticFilters = true; var ignoreGenericCosmeticFilters = false; var externalLists = ''; -var cacheWasPurged = false; -var needUpdate = false; -var hasCachedContent = false; /******************************************************************************/ var onMessage = function(msg) { switch ( msg.what ) { + case 'assetUpdated': + updateAssetStatus(msg); + break; case 'staticFilteringDataChanged': renderFilterLists(); break; - - case 'forceUpdateAssetsProgress': - renderBusyOverlay(true, msg.progress); - if ( msg.done ) { - messaging.send('dashboard', { what: 'reloadAllFilters' }); - } - break; - default: break; } @@ -69,8 +60,6 @@ var renderNumber = function(value) { /******************************************************************************/ -// TODO: get rid of background page dependencies - var renderFilterLists = function() { var listGroupTemplate = uDom('#templates .groupEntry'); var listEntryTemplate = uDom('#templates .listEntry'); @@ -80,9 +69,6 @@ var renderFilterLists = function() { // Assemble a pretty blacklist name if possible var listNameFromListKey = function(listKey) { - if ( listKey === listDetails.userFiltersPath ) { - return userListName; - } var list = listDetails.current[listKey] || listDetails.available[listKey]; var listTitle = list ? list.title : ''; if ( listTitle === '' ) { @@ -124,39 +110,28 @@ var renderFilterLists = function() { .replace('{{total}}', !isNaN(+entry.entryCount) ? renderNumber(entry.entryCount) : '?'); elem.text(text); - // https://github.com/gorhill/uBlock/issues/78 - // Badge for non-secure connection - var remoteURL = listKey; - if ( remoteURL.lastIndexOf('http:', 0) !== 0 ) { - remoteURL = entry.homeURL || ''; - } - if ( remoteURL.lastIndexOf('http:', 0) === 0 ) { - li.descendants('span.status.unsecure').css('display', ''); - } - // https://github.com/chrisaljoudi/uBlock/issues/104 var asset = listDetails.cache[listKey] || {}; + // https://github.com/gorhill/uBlock/issues/78 + // Badge for non-secure connection + var remoteURL = asset.remoteURL; + li.toggleClass( + 'unsecure', + typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0 + ); + // Badge for update status - if ( entry.off !== true ) { - if ( asset.repoObsolete ) { - li.descendants('span.status.new').css('display', ''); - needUpdate = true; - } else if ( asset.cacheObsolete ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; - } else if ( entry.external && !asset.cached ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; - } - } + li.toggleClass('obsolete', entry.off !== true && asset.obsolete === true); + + // Badge for cache status + li.toggleClass('cached', asset.cached === true && asset.writeTime > 0); - // In cache if ( asset.cached ) { - elem = li.descendants('span.status.purge'); - elem.css('display', ''); - elem.attr('title', lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.lastModified))); - hasCachedContent = true; + elem = li.descendants('.status.purge').attr( + 'title', + lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime)) + ); } return li; }; @@ -219,8 +194,6 @@ var renderFilterLists = function() { listDetails = details; parseCosmeticFilters = details.parseCosmeticFilters; ignoreGenericCosmeticFilters = details.ignoreGenericCosmeticFilters; - needUpdate = false; - hasCachedContent = false; // Visually split the filter lists in purpose-based groups var ulLists = uDom('#lists').empty(), liGroup; @@ -253,17 +226,17 @@ var renderFilterLists = function() { ulLists.append(liFromListGroup(groupKey, groups[groupKey])); } + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); + uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); + uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); + uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); uDom('#listsOfBlockedHostsPrompt').text( vAPI.i18n('3pListsOfBlockedHostsPrompt') .replace('{{netFilterCount}}', renderNumber(details.netFilterCount)) .replace('{{cosmeticFilterCount}}', renderNumber(details.cosmeticFilterCount)) ); - uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); - uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); - uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); renderWidgets(); - renderBusyOverlay(details.manualUpdate, details.manualUpdateProgress); }; messaging.send('dashboard', { what: 'getLists' }, onListsReceived); @@ -271,33 +244,19 @@ var renderFilterLists = function() { /******************************************************************************/ -// Progress must be normalized to [0, 1], or can be undefined. +// This is to give a visual hint that the selection of blacklists has changed. -var renderBusyOverlay = function(state, progress) { - progress = progress || {}; - var showProgress = typeof progress.value === 'number'; - if ( showProgress ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:first-child').css( - 'width', - (progress.value * 100).toFixed(1) + '%' - ); - var text = progress.text || ''; - if ( text !== '' ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:last-child').text(text); - } - } - uDom('#busyOverlay > div:nth-of-type(2)').css('display', showProgress ? '' : 'none'); - uDom('body').toggleClass('busy', !!state); +var renderWidgets = function() { + uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); + uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null); }; /******************************************************************************/ -// This is to give a visual hint that the selection of blacklists has changed. - -var renderWidgets = function() { - uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); - uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged()); - uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent); +var updateAssetStatus = function(details) { + var li = uDom('#lists .listEntry a[data-listkey="' + details.key + '"]').ancestors('.listEntry'); + li.toggleClass('obsolete', !details.cached); + li.toggleClass('cached', details.cached); }; /******************************************************************************/ @@ -307,41 +266,30 @@ var renderWidgets = function() { var listsSelectionChanged = function() { if ( listDetails.parseCosmeticFilters !== parseCosmeticFilters || - listDetails.parseCosmeticFilters && listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters + listDetails.parseCosmeticFilters && + listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters ) { return true; } - if ( cacheWasPurged ) { - return true; - } - - var availableLists = listDetails.available; - var currentLists = listDetails.current; - var location, availableOff, currentOff; + var availableLists = listDetails.available, + currentLists = listDetails.current, + listKey, availableOff, currentOff; - // This check existing entries - for ( location in availableLists ) { - if ( availableLists.hasOwnProperty(location) === false ) { - continue; - } - availableOff = availableLists[location].off === true; - currentOff = currentLists[location] === undefined || currentLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; - } + // This checks existing entries + for ( listKey in availableLists ) { + if ( availableLists.hasOwnProperty(listKey) === false ) { continue; } + availableOff = availableLists[listKey].off === true; + currentOff = currentLists[listKey] === undefined || currentLists[listKey].off === true; + if ( availableOff !== currentOff ) { return true; } } - // This check removed entries - for ( location in currentLists ) { - if ( currentLists.hasOwnProperty(location) === false ) { - continue; - } - currentOff = currentLists[location].off === true; - availableOff = availableLists[location] === undefined || availableLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; - } + // This checks removed entries + for ( listKey in currentLists ) { + if ( currentLists.hasOwnProperty(listKey) === false ) { continue; } + currentOff = currentLists[listKey].off === true; + availableOff = availableLists[listKey] === undefined || availableLists[listKey].off === true; + if ( availableOff !== currentOff ) { return true; } } return false; @@ -349,22 +297,10 @@ var listsSelectionChanged = function() { /******************************************************************************/ -// Return whether content need update. - -var listsContentChanged = function() { - return needUpdate; -}; - -/******************************************************************************/ - var onListCheckboxChanged = function() { var href = uDom(this).parent().descendants('a').first().attr('data-listkey'); - if ( typeof href !== 'string' ) { - return; - } - if ( listDetails.available[href] === undefined ) { - return; - } + if ( typeof href !== 'string' ) { return; } + if ( listDetails.available[href] === undefined ) { return; } listDetails.available[href].off = !this.checked; renderWidgets(); }; @@ -374,12 +310,10 @@ var onListCheckboxChanged = function() { var onPurgeClicked = function() { var button = uDom(this); var li = button.parent(); - var href = li.descendants('a').first().attr('data-listkey'); - if ( !href ) { - return; - } + var listKey = li.descendants('a').first().attr('data-listkey'); + if ( !listKey ) { return; } - messaging.send('dashboard', { what: 'purgeCache', path: href }); + messaging.send('dashboard', { what: 'purgeCache', path: listKey }); button.remove(); // If the cached version is purged, the installed version must be assumed @@ -387,18 +321,14 @@ var onPurgeClicked = function() { // https://github.com/gorhill/uBlock/issues/1733 // An external filter list must not be marked as obsolete, they will always // be fetched anyways if there is no cached copy. - var entry = listDetails.current && listDetails.current[href]; - if ( entry && entry.off !== true && /^[a-z]+:\/\//.test(href) === false ) { - if ( typeof entry.homeURL !== 'string' || entry.homeURL === '' ) { - li.descendants('span.status.new').css('display', ''); - } else { - li.descendants('span.status.obsolete').css('display', ''); - } - needUpdate = true; + var entry = listDetails.current && listDetails.current[listKey]; + if ( entry && entry.off !== true ) { + li.addClass('obsolete'); + uDom('#buttonUpdate').removeClass('disabled'); } + li.removeClass('cached'); if ( li.descendants('input').first().prop('checked') ) { - cacheWasPurged = true; renderWidgets(); } }; @@ -419,22 +349,21 @@ var selectFilterLists = function(callback) { }); // Filter lists - var switches = []; - var lis = uDom('#lists .listEntry'), li; - var i = lis.length; + var listKeys = [], + lis = uDom('#lists .listEntry'), li, + i = lis.length; while ( i-- ) { li = lis.at(i); - switches.push({ - location: li.descendants('a').attr('data-listkey'), - off: li.descendants('input').prop('checked') === false - }); + if ( li.descendants('input').prop('checked') ) { + listKeys.push(li.descendants('a').attr('data-listkey')); + } } messaging.send( 'dashboard', { what: 'selectFilterLists', - switches: switches + keys: listKeys }, callback ); @@ -444,49 +373,38 @@ var selectFilterLists = function(callback) { var buttonApplyHandler = function() { uDom('#buttonApply').removeClass('enabled'); - - renderBusyOverlay(true); - var onSelectionDone = function() { messaging.send('dashboard', { what: 'reloadAllFilters' }); }; - selectFilterLists(onSelectionDone); - - cacheWasPurged = false; }; /******************************************************************************/ -var buttonUpdateHandler = function() { +var buttonUpdateHandler = function(ev) { uDom('#buttonUpdate').removeClass('enabled'); - - if ( needUpdate ) { - renderBusyOverlay(true); - - var onSelectionDone = function() { - messaging.send('dashboard', { what: 'forceUpdateAssets' }); - }; - - selectFilterLists(onSelectionDone); - - cacheWasPurged = false; - } + var onSelectionDone = function() { + messaging.send('dashboard', { + what: 'forceUpdateAssets', + fast: ev.ctrlKey + }); + uDom('#buttonUpdate').addClass('disabled'); + }; + selectFilterLists(onSelectionDone); }; /******************************************************************************/ -var buttonPurgeAllHandler = function() { +var buttonPurgeAllHandler = function(ev) { uDom('#buttonPurgeAll').removeClass('enabled'); - - renderBusyOverlay(true); - - var onCompleted = function() { - cacheWasPurged = true; - renderFilterLists(); - }; - - messaging.send('dashboard', { what: 'purgeAllCaches' }, onCompleted); + messaging.send( + 'dashboard', + { + what: 'purgeAllCaches', + hard: ev.ctrlKey && ev.shiftKey + }, + renderFilterLists + ); }; /******************************************************************************/ diff --git a/src/js/assets.js b/src/js/assets.js index 88909948b5b81..91c91b2072721 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,305 +19,47 @@ Home: https://github.com/gorhill/uBlock */ -/* global YaMD5 */ - 'use strict'; -/******************************************************************************* - -File system structure: - assets - ublock - ... - thirdparties - ... - user - filters.txt - ... - -*/ - /******************************************************************************/ -// Low-level asset files manager - µBlock.assets = (function() { /******************************************************************************/ -var oneSecond = 1000; -var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -var oneDay = 24 * oneHour; - -/******************************************************************************/ - -var projectRepositoryRoot = µBlock.projectServerRoot; -var assetsRepositoryRoot = 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/'; -var nullFunc = function() {}; -var reIsExternalPath = /^(file|ftps?|https?|resource):\/\//; -var reIsUserPath = /^assets\/user\//; -var reIsCachePath = /^cache:\/\//; -var lastRepoMetaTimestamp = 0; -var lastRepoMetaIsRemote = false; -var refreshRepoMetaPeriod = 5 * oneHour; +var reIsExternalPath = /^(?:file|ftps?|https?|resource):\/\//; +var reIsUserAsset = /^user-/; var errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); var xhrTimeout = vAPI.localStorage.getItem('xhrTimeout') || 30000; -var onAssetRemovedListener = null; -var exports = { - autoUpdate: true, - autoUpdateDelay: 4 * oneDay, - - // https://github.com/chrisaljoudi/uBlock/issues/426 - remoteFetchBarrier: 0 +var api = { }; /******************************************************************************/ -var AssetEntry = function() { - this.localChecksum = ''; - this.repoChecksum = ''; - this.expireTimestamp = 0; -}; - -var RepoMetadata = function() { - this.entries = {}; - this.waiting = []; -}; - -var repoMetadata = null; - -// We need these to persist beyond repoMetaData -var homeURLs = {}; +var observers = []; -/******************************************************************************/ - -var stringIsNotEmpty = function(s) { - return typeof s === 'string' && s !== ''; +api.addObserver = function(observer) { + if ( observers.indexOf(observer) === -1 ) { + observers.push(observer); + } }; -/******************************************************************************/ - -var cacheIsObsolete = function(t) { - return typeof t !== 'number' || (Date.now() - t) >= exports.autoUpdateDelay; +api.removeObserver = function(observer) { + var pos; + while ( (pos = observers.indexOf(observer)) !== -1 ) { + observers.splice(pos, 1); + } }; -/******************************************************************************/ - -var cachedAssetsManager = (function() { - var exports = {}; - var entries = null; - var cachedAssetPathPrefix = 'cached_asset_content://'; - - var getEntries = function(callback) { - if ( entries !== null ) { - callback(entries); - return; - } - // Flush cached non-user assets if these are from a prior version. - // https://github.com/gorhill/httpswitchboard/issues/212 - var onLastVersionRead = function(store) { - var currentVersion = vAPI.app.version; - var lastVersion = store.extensionLastVersion || '0.0.0.0'; - if ( currentVersion !== lastVersion ) { - vAPI.cacheStorage.set({ 'extensionLastVersion': currentVersion }); - } - callback(entries); - }; - var onLoaded = function(bin) { - // https://github.com/gorhill/httpswitchboard/issues/381 - // Maybe the index was requested multiple times and already - // fetched by one of the occurrences. - if ( entries === null ) { - var lastError = vAPI.lastError(); - if ( lastError ) { - console.error( - 'µBlock> cachedAssetsManager> getEntries():', - lastError.message - ); - } - entries = bin.cached_asset_entries || {}; - } - vAPI.cacheStorage.get('extensionLastVersion', onLastVersionRead); - }; - vAPI.cacheStorage.get('cached_asset_entries', onLoaded); - }; - exports.entries = getEntries; - - exports.load = function(path, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - 'path': path, - 'content': '' - }; - var cachedContentPath = cachedAssetPathPrefix + path; - var onLoaded = function(bin) { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µBlock> cachedAssetsManager.load():', details.error); - cbError(details); - return; - } - // Not sure how this can happen, but I've seen it happen. It could - // be because the save occurred while I was stepping in the code - // though, which means it would not occur during normal operation. - // Still, just to be safe. - if ( stringIsNotEmpty(bin[cachedContentPath]) === false ) { - exports.remove(path); - details.error = 'Error: not found'; - cbError(details); - return; - } - details.content = bin[cachedContentPath]; - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries[path] === undefined ) { - details.error = 'Error: not found'; - cbError(details); - return; - } - vAPI.cacheStorage.get(cachedContentPath, onLoaded); - }; - getEntries(onEntries); - }; - - exports.save = function(path, content, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - path: path, - content: content - }; - if ( content === '' ) { - exports.remove(path); - cbSuccess(details); - return; +var fireNotification = function(topic, details) { + var result; + for ( var i = 0; i < observers.length; i++ ) { + if ( observers[i](topic, details) === false ) { + result = false; } - var cachedContentPath = cachedAssetPathPrefix + path; - var bin = {}; - bin[cachedContentPath] = content; - var removedItems = []; - var onSaved = function() { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µBlock> cachedAssetsManager.save():', details.error); - cbError(details); - return; - } - // Saving over an existing item must be seen as removing an - // existing item and adding a new one. - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener(removedItems); - } - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries.hasOwnProperty(path) ) { - removedItems.push(path); - } - entries[path] = Date.now(); - bin.cached_asset_entries = entries; - vAPI.cacheStorage.set(bin, onSaved); - }; - getEntries(onEntries); - }; - - exports.remove = function(pattern, before) { - var onEntries = function(entries) { - var keystoRemove = []; - var removedItems = []; - var paths = Object.keys(entries); - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( typeof pattern === 'string' && path !== pattern ) { - continue; - } - if ( pattern instanceof RegExp && !pattern.test(path) ) { - continue; - } - if ( typeof before === 'number' && entries[path] >= before ) { - continue; - } - removedItems.push(path); - keystoRemove.push(cachedAssetPathPrefix + path); - delete entries[path]; - } - if ( keystoRemove.length ) { - vAPI.cacheStorage.remove(keystoRemove); - vAPI.cacheStorage.set({ 'cached_asset_entries': entries }); - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener(removedItems); - } - } - }; - getEntries(onEntries); - }; - - exports.removeAll = function(callback) { - var onEntries = function() { - // Careful! do not remove 'assets/user/' - exports.remove(/^https?:\/\/[a-z0-9]+/); - exports.remove(/^assets\/(ublock|thirdparties)\//); - exports.remove(/^cache:\/\//); - exports.remove('assets/checksums.txt'); - if ( typeof callback === 'function' ) { - callback(null); - } - }; - getEntries(onEntries); - }; - - exports.rmrf = function() { - exports.remove(/./); - }; - - exports.exists = function(path) { - return entries !== null && entries.hasOwnProperty(path); - }; - - getEntries(function(){}); - - return exports; -})(); - -/******************************************************************************/ - -var toRepoURL = function(path) { - if ( path.startsWith('assets/ublock/filter-lists.json') ) { - return projectRepositoryRoot + path; - } - - if ( path.startsWith('assets/checksums.txt') ) { - return path.replace( - /^assets\/checksums.txt/, - assetsRepositoryRoot + 'checksums/ublock0.txt' - ); - } - - if ( path.startsWith('assets/thirdparties/') ) { - return path.replace( - /^assets\/thirdparties\//, - assetsRepositoryRoot + 'thirdparties/' - ); - } - - if ( path.startsWith('assets/ublock/') ) { - return path.replace( - /^assets\/ublock\//, - assetsRepositoryRoot + 'filters/' - ); } - - // At this point, `path` is assumed to point to a resource specific to - // this project. - return projectRepositoryRoot + path; + return result; }; /******************************************************************************/ @@ -353,6 +95,7 @@ var getTextFileFromURL = function(url, onLoad, onError) { var onErrorReceived = function() { this.onload = this.onerror = this.ontimeout = null; + µBlock.logger.writeOne('', 'error', errorCantConnectTo.replace('{{msg}}', '')); onError.call(this); }; @@ -373,1289 +116,724 @@ var getTextFileFromURL = function(url, onLoad, onError) { } }; -/******************************************************************************/ +/******************************************************************************* + + The purpose of the asset source registry is to keep key detail information + about an asset: + - Where to load it from: this may consist of one or more URLs, either local + or remote. + - After how many days an asset should be deemed obsolete -- i.e. in need of + an update. + - The origin and type of an asset. + - The last time an asset was registered. + +**/ + +var assetSourceRegistryStatus, + assetSourceRegistry = Object.create(null); + +var registerAssetSource = function(assetKey, details) { + var contentURL = details.contentURL || []; + if ( typeof contentURL === 'string' ) { + contentURL = [ contentURL ]; + } + var updateAfter = details.updateAfter || 11; + var submitter = details.submitter || undefined; + var entry = assetSourceRegistry[assetKey]; + if ( entry === undefined ) { + entry = Object.create(null); + } + var remoteURLCount = 0; + for ( var i = 0; i < contentURL.length; i++ ) { + if ( reIsExternalPath.test(contentURL[i]) ) { + remoteURLCount += 1; + } + } + entry.contentURL = contentURL; + entry.hasLocalURL = remoteURLCount !== contentURL.length; + entry.hasRemoteURL = remoteURLCount !== 0; + entry.updateAfter = updateAfter; + entry.submitter = submitter; + if ( entry.submitter ) { + entry.submitTime = Date.now(); // To detect stale entries + } + assetSourceRegistry[assetKey] = entry; +}; + +var registerAssetError = function(assetKey, details) { + getAssetSourceRegistry(function() { + var entry = assetSourceRegistry[assetKey]; + if ( entry === undefined ) { return; } + entry.error = details; + }); +}; -var updateLocalChecksums = function() { - var localChecksums = []; - var entries = repoMetadata.entries; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; +var saveAssetSourceRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry }); + }; + return function(lazily) { + if ( timer !== undefined ) { + clearTimeout(timer); } - entry = entries[path]; - if ( entry.localChecksum !== '' ) { - localChecksums.push(entry.localChecksum + ' ' + path); + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); } + }; +})(); + +var getAssetSourceRegistry = function(callback) { + // Already loaded. + if ( assetSourceRegistryStatus === 'ready' ) { + callback(assetSourceRegistry); + return; } - cachedAssetsManager.save('assets/checksums.txt', localChecksums.join('\n')); + + // Being loaded. + if ( Array.isArray(assetSourceRegistryStatus) ) { + assetSourceRegistryStatus.push(callback); + return; + } + + // Not loaded: load it. + assetSourceRegistryStatus = [ callback ]; + + var registryReady = function() { + var callers = assetSourceRegistryStatus; + assetSourceRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetSourceRegistry); + } + }; + + // First-install case. + var createRegistry = function() { + getTextFileFromURL(vAPI.getURL('assets/assets.json'), function() { + var assetDict; + try { + assetDict = JSON.parse(this.responseText); + } catch (ex) { + } + for ( var assetKey in assetDict ) { + registerAssetSource(assetKey, assetDict[assetKey]); + } + saveAssetSourceRegistry(); + registryReady(); + }); + }; + + vAPI.cacheStorage.get('assetSourceRegistry', function(bin) { + if ( !bin || !bin.assetSourceRegistry ) { + createRegistry(); + return; + } + assetSourceRegistry = bin.assetSourceRegistry; + registryReady(); + }); }; -/******************************************************************************/ +//var updateAssetSourceRegistry = function(raw) {}; + +api.getAssetSourceRegistry = function(callback) { + getAssetSourceRegistry(function() { + callback(JSON.parse(JSON.stringify(assetSourceRegistry))); + }); +}; + +api.registerAssetSource = function(assetKey, details) { + getAssetSourceRegistry(function() { + registerAssetSource(assetKey, details); + saveAssetSourceRegistry(); + }); +}; + +/******************************************************************************* -// Gather meta data of all assets. + The purpose of the asset cache registry is to keep track of all assets + which have been persisted into the local cache. -var getRepoMetadata = function(callback) { - callback = callback || nullFunc; +**/ - // https://github.com/chrisaljoudi/uBlock/issues/515 - // Handle re-entrancy here, i.e. we MUST NOT tamper with the waiting list - // of callers, if any, except to add one at the end of the list. - if ( repoMetadata !== null && repoMetadata.waiting.length !== 0 ) { - repoMetadata.waiting.push(callback); +var assetCacheRegistryStatus, + assetCacheRegistry = {}; + +var getAssetCacheRegistry = function(callback) { + // Already loaded. + if ( assetCacheRegistryStatus === 'ready' ) { + callback(assetCacheRegistry); return; } - if ( exports.remoteFetchBarrier === 0 && lastRepoMetaIsRemote === false ) { - lastRepoMetaTimestamp = 0; - } - if ( (Date.now() - lastRepoMetaTimestamp) >= refreshRepoMetaPeriod ) { - repoMetadata = null; - } - if ( repoMetadata !== null ) { - callback(repoMetadata); + // Being loaded. + if ( Array.isArray(assetCacheRegistryStatus) ) { + assetCacheRegistryStatus.push(callback); return; } - lastRepoMetaTimestamp = Date.now(); - lastRepoMetaIsRemote = exports.remoteFetchBarrier === 0; - - var defaultChecksums; - var localChecksums; - var repoChecksums; + // Not loaded: load it. + assetCacheRegistryStatus = [ callback ]; - var checksumsReceived = function() { - if ( - defaultChecksums === undefined || - localChecksums === undefined || - repoChecksums === undefined - ) { - return; + var registryReady = function() { + var callers = assetCacheRegistryStatus; + assetCacheRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetCacheRegistry); } - // Remove from cache assets which no longer exist in the repo - var entries = repoMetadata.entries; - var checksumsChanged = false; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entry = entries[path]; - // https://github.com/gorhill/uBlock/issues/760 - // If the resource does not have a cached instance, we must reset - // the checksum to its value at install time. - if ( - stringIsNotEmpty(defaultChecksums[path]) && - entry.localChecksum !== defaultChecksums[path] && - cachedAssetsManager.exists(path) === false - ) { - entry.localChecksum = defaultChecksums[path]; - checksumsChanged = true; - } - // If repo checksums could not be fetched, assume no change. - // https://github.com/gorhill/uBlock/issues/602 - // Added: if repo checksum is that of the empty string, - // assume no change - if ( - repoChecksums === '' || - entry.repoChecksum === 'd41d8cd98f00b204e9800998ecf8427e' - ) { - entry.repoChecksum = entry.localChecksum; - } - if ( entry.repoChecksum !== '' || entry.localChecksum === '' ) { - continue; - } - checksumsChanged = true; - cachedAssetsManager.remove(path); - entry.localChecksum = ''; + }; + + vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { + if ( bin && bin.assetCacheRegistry ) { + assetCacheRegistry = bin.assetCacheRegistry; } - if ( checksumsChanged ) { - updateLocalChecksums(); + registryReady(); + }); +}; + +var saveAssetCacheRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry }); + }; + return function(lazily) { + if ( timer !== undefined ) { + clearTimeout(timer); } - // Notify all waiting callers - // https://github.com/chrisaljoudi/uBlock/issues/515 - // VERY IMPORTANT: because of re-entrancy, we MUST: - // - process the waiting callers in a FIFO manner - // - not cache repoMetadata.waiting.length, we MUST use the live - // value, because it can change while looping - // - not change the waiting list until they are all processed - for ( var i = 0; i < repoMetadata.waiting.length; i++ ) { - repoMetadata.waiting[i](repoMetadata); + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); } - repoMetadata.waiting.length = 0; }; +})(); - var validateChecksums = function(details) { - if ( details.error || details.content === '' ) { - return ''; - } - if ( /^(?:[0-9a-f]{32}\s+\S+(?:\s+|$))+/.test(details.content) === false ) { - return ''; +var assetCacheRead = function(assetKey, callback) { + var internalKey = 'cache/' + assetKey; + + var reportBack = function(content, err) { + var details = { assetKey: assetKey, content: content }; + if ( err ) { details.error = err; } + callback(details); + }; + + var onAssetRead = function(bin) { + if ( !bin || !bin[internalKey] ) { + return reportBack('', 'E_NOTFOUND'); } - // https://github.com/gorhill/uBlock/issues/602 - // External filter lists are not meant to appear in checksums.txt. - // TODO: remove this code once v1.1.0.0 is everywhere. - var out = []; - var listMap = µBlock.oldListToNewListMap; - var lines = details.content.split(/\s*\n\s*/); - var line, matches; - for ( var i = 0; i < lines.length; i++ ) { - line = lines[i]; - matches = line.match(/^[0-9a-f]+ (.+)$/); - if ( matches === null || listMap.hasOwnProperty(matches[1]) ) { - continue; - } - out.push(line); + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + return reportBack('', 'E_NOTFOUND'); } - return out.join('\n'); + entry.readTime = Date.now(); + saveAssetCacheRegistry(true); + reportBack(bin[internalKey]); }; - var parseChecksums = function(text, eachFn) { - var lines = text.split(/\n+/); - var i = lines.length; - var fields; - while ( i-- ) { - fields = lines[i].trim().split(/\s+/); - if ( fields.length !== 2 ) { - continue; - } - eachFn(fields[1], fields[0]); + var onReady = function() { + vAPI.cacheStorage.get(internalKey, onAssetRead); + }; + + getAssetCacheRegistry(onReady); +}; + +var assetCacheWrite = function(assetKey, details, callback) { + var internalKey = 'cache/' + assetKey; + var content = ''; + if ( typeof details === 'string' ) { + content = details; + } else if ( details instanceof Object ) { + content = details.content || ''; + } + + if ( content === '' ) { + return assetCacheRemove(assetKey, callback); + } + + var reportBack = function(content) { + var details = { assetKey: assetKey, content: content }; + if ( typeof callback === 'function' ) { + callback(details); } + fireNotification('after-asset-updated', details); }; - var onLocalChecksumsLoaded = function(details) { - var entries = repoMetadata.entries; - var processChecksum = function(path, checksum) { - if ( entries.hasOwnProperty(path) === false ) { - entries[path] = new AssetEntry(); - } - entries[path].localChecksum = checksum; - }; - if ( (localChecksums = validateChecksums(details)) ) { - parseChecksums(localChecksums, processChecksum); + var onReady = function() { + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + entry = assetCacheRegistry[assetKey] = {}; + } + entry.writeTime = entry.readTime = Date.now(); + if ( details instanceof Object && typeof details.url === 'string' ) { + entry.remoteURL = details.url; } - checksumsReceived(); + var bin = { assetCacheRegistry: assetCacheRegistry }; + bin[internalKey] = content; + vAPI.cacheStorage.set(bin); + reportBack(content); }; + getAssetCacheRegistry(onReady); +}; - var onRepoChecksumsLoaded = function(details) { - var entries = repoMetadata.entries; - var processChecksum = function(path, checksum) { - if ( entries.hasOwnProperty(path) === false ) { - entries[path] = new AssetEntry(); +var assetCacheRemove = function(pattern, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + removedEntries = [], + removedContent = []; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { + continue; + } + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; } - entries[path].repoChecksum = checksum; - }; - if ( (repoChecksums = validateChecksums(details)) ) { - parseChecksums(repoChecksums, processChecksum); + removedEntries.push(assetKey); + removedContent.push('cache/' + assetKey); + delete cacheDict[assetKey]; + } + if ( removedContent.length !== 0 ) { + vAPI.cacheStorage.remove(removedContent); + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); + } + if ( typeof callback === 'function' ) { + callback(); + } + for ( var i = 0; i < removedEntries.length; i++ ) { + fireNotification('after-asset-updated', { assetKey: removedEntries[i] }); } - checksumsReceived(); }; - // https://github.com/gorhill/uBlock/issues/760 - // We need the checksum values at install time, because some resources - // may have been purged, in which case the checksum must be reset to the - // value at install time. - var onDefaultChecksumsLoaded = function() { - defaultChecksums = Object.create(null); - var processChecksum = function(path, checksum) { - defaultChecksums[path] = checksum; - }; - parseChecksums(this.responseText || '', processChecksum); - checksumsReceived(); + getAssetCacheRegistry(onReady); +}; + +var assetCacheMarkAsDirty = function(pattern, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + cacheEntry, + mustSave = false; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { + continue; + } + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; + } + cacheEntry = cacheDict[assetKey]; + if ( !cacheEntry.writeTime ) { continue; } + cacheDict[assetKey].writeTime = 0; + mustSave = true; + } + if ( mustSave ) { + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); + } + if ( typeof callback === 'function' ) { + callback(); + } }; - repoMetadata = new RepoMetadata(); - repoMetadata.waiting.push(callback); - readRepoFile('assets/checksums.txt', onRepoChecksumsLoaded); - getTextFileFromURL(vAPI.getURL('assets/checksums.txt'), onDefaultChecksumsLoaded); - readLocalFile('assets/checksums.txt', onLocalChecksumsLoaded); + getAssetCacheRegistry(onReady); }; -// https://www.youtube.com/watch?v=-t3WYfgM4x8 - /******************************************************************************/ -exports.setHomeURL = function(path, homeURL) { - if ( typeof homeURL !== 'string' || homeURL === '' ) { - return; - } - homeURLs[path] = homeURL; +var stringIsNotEmpty = function(s) { + return typeof s === 'string' && s !== ''; }; -/******************************************************************************/ +/******************************************************************************* -// Get a local asset, do not look-up repo or remote location if local asset -// is not found. + TODO: This will be removed when I am confident all users have moved + to a version of uBO which does not require the old way to persist cached + assets. -var readLocalFile = function(path, callback) { - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; +**/ + +(function() { + vAPI.cacheStorage.get('cached_asset_entries', function(bin) { + var entries = bin && bin['cached_asset_entries']; + if ( !entries ) { return; } + var keystoRemove = [], + paths = Object.keys(entries), + i = paths.length, + path; + while ( i-- ) { + path = paths[i]; + if ( path === 'assets/user/filters.txt' ) { continue; } + keystoRemove.push('cached_asset_content://' + path); } - callback(details); - }; + keystoRemove.push('extensionLastVersion'); + keystoRemove.push('cached_asset_entries'); + vAPI.cacheStorage.remove(keystoRemove); + }); +})(); - var onInstallFileLoaded = function() { - //console.log('µBlock> readLocalFile("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; +/******************************************************************************* - var onInstallFileError = function() { - console.error('µBlock> readLocalFile("%s") / onInstallFileError()', path); - reportBack('', 'Error'); - }; + User assets are NOT persisted in the cache storage. User assets are + recognized by the asset key which always starts with 'user-'. - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readLocalFile("%s") / onCachedContentLoaded()', path); - reportBack(details.content); + TODO: Revisit when I am confident all users have moved to a version of uBO + which does not require the old way to persist user assets. + +**/ + +var readUserAsset = function(assetKey, callback) { + // TODO: remove old cruft when confident all users moved to uBO 1.11.0+. + var reportBack = function(content) { + callback({ assetKey: assetKey, content: content }); }; - var onCachedContentError = function(details) { - //console.error('µBlock> readLocalFile("%s") / onCachedContentError()', path); - if ( reIsExternalPath.test(path) ) { - reportBack('', 'Error: asset not found'); - return; + var onLoaded = function(bin) { + if ( !bin ) { + return reportBack(''); } - // It's ok for user data to not be found - if ( reIsUserPath.test(path) ) { - reportBack(''); - return; + var content = ''; + if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) { + content = bin['cached_asset_content://assets/user/filters.txt']; + vAPI.cacheStorage.remove('cached_asset_content://assets/user/filters.txt'); + } + if ( typeof bin['assets/user/filters.txt'] === 'string' ) { + content = bin['assets/user/filters.txt']; + vAPI.storage.remove('assets/user/filters.txt'); } - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); + if ( typeof bin[assetKey] === 'string' ) { + content = bin[assetKey]; + } + return reportBack(content); }; - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + vAPI.storage.get( + [ + assetKey, + 'assets/user/filters.txt', + 'cached_asset_content://assets/user/filters.txt' + ], + onLoaded + ); }; -// https://www.youtube.com/watch?v=r9KVpuFPtHc +var saveUserAsset = function(assetKey, content, callback) { + var bin = {}; + bin[assetKey] = content; + var onSaved = function() { + if ( callback instanceof Function ) { + callback({ assetKey: assetKey, content: content }); + } + }; + vAPI.storage.set(bin, onSaved); +}; /******************************************************************************/ -// Get the repository copy of a built-in asset. - -var readRepoFile = function(path, callback) { - // https://github.com/chrisaljoudi/uBlock/issues/426 - if ( exports.remoteFetchBarrier !== 0 ) { - readLocalFile(path, callback); +api.get = function(assetKey, callback) { + if ( reIsUserAsset.test(assetKey) ) { + readUserAsset(assetKey, callback); return; } + var assetDetails = {}, + contentURLs, + contentURL; + var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content, - 'error': err - }; + var details = { assetKey: assetKey, content: content }; + if ( err ) { + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; + } callback(details); }; - var repositoryURL = toRepoURL(path); + var onContentNotLoaded = function() { + var isExternal; + while ( (contentURL = contentURLs.shift()) ) { + isExternal = reIsExternalPath.test(contentURL); + if ( isExternal === false || assetDetails.hasLocalURL !== true ) { + break; + } + } + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); + } + getTextFileFromURL( + isExternal ? contentURL : vAPI.getURL(contentURL), + onContentLoaded, + onContentNotLoaded + ); + }; - var onRepoFileLoaded = function() { - //console.log('µBlock> readRepoFile("%s") / onRepoFileLoaded()', path); - // https://github.com/gorhill/httpswitchboard/issues/263 - if ( this.status === 200 ) { - reportBack(this.responseText); - } else { - reportBack('', 'Error: ' + this.statusText); + var onContentLoaded = function() { + if ( stringIsNotEmpty(this.responseText) === false ) { + onContentNotLoaded(); + return; } + if ( reIsExternalPath.test(contentURL) ) { + assetCacheWrite(assetKey, { + content: this.responseText, + url: contentURL + }); + } + reportBack(this.responseText); }; - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - reportBack('', 'Error'); + var onCachedContentLoaded = function(details) { + if ( details.content !== '' ) { + return reportBack(details.content); + } + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; + } + onContentNotLoaded(); + }); }; - // '_=...' is to skip browser cache - getTextFileFromURL( - repositoryURL + '?_=' + Date.now(), - onRepoFileLoaded, - onRepoFileError - ); + assetCacheRead(assetKey, onCachedContentLoaded); }; /******************************************************************************/ -// An asset from an external source with a copy shipped with the extension: -// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL -// External --> -// Repository --> has checksum (to detect need for update only) -// Cache --> has expiration timestamp (in cache) -// Local --> install time version - -var readRepoCopyAsset = function(path, callback) { - var assetEntry; - var homeURL = homeURLs[path]; +var getRemote = function(assetKey, callback) { + var assetDetails = {}, + contentURLs, + contentURL; var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; + var details = { assetKey: assetKey, content: content }; if ( err ) { - details.error = err; + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; } callback(details); }; - var updateChecksum = function() { - if ( assetEntry !== undefined && assetEntry.repoChecksum !== assetEntry.localChecksum ) { - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); + var onRemoteContentLoaded = function() { + if ( stringIsNotEmpty(this.responseText) === false ) { + registerAssetError(assetKey, { time: Date.now(), error: 'No content' }); + tryLoading(); + return; } - }; - - var onInstallFileLoaded = function() { - //console.log('µBlock> readRepoCopyAsset("%s") / onInstallFileLoaded()', path); + assetCacheWrite(assetKey, { + content: this.responseText, + url: contentURL + }); + registerAssetError(assetKey); reportBack(this.responseText); }; - var onInstallFileError = function() { - console.error('µBlock> readRepoCopyAsset("%s") / onInstallFileError():', path, this.statusText); - reportBack('', 'Error'); + var onRemoteContentError = function() { + registerAssetError(assetKey, { time: Date.now(), error: this.statusText }); + tryLoading(); }; - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileLoaded()', path); - reportBack(details.content); + var tryLoading = function() { + while ( (contentURL = contentURLs.shift()) ) { + if ( reIsExternalPath.test(contentURL) ) { break; } + } + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); + } + getTextFileFromURL(contentURL, onRemoteContentLoaded, onRemoteContentError); }; - var onCachedContentError = function(details) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileError()', path); - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); - }; + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; + } + tryLoading(); + }); +}; - var repositoryURL = toRepoURL(path); - var repositoryURLSkipCache = repositoryURL + '?_=' + Date.now(); +/******************************************************************************/ - var onRepoFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s"): error', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - //console.log('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); - }; +api.put = function(assetKey, content, callback) { + if ( reIsUserAsset.test(assetKey) ) { + return saveUserAsset(assetKey, content, callback); + } + assetCacheWrite(assetKey, content, callback); +}; - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; +/******************************************************************************/ - var onHomeFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s"): no response', path, homeURL); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); +api.metadata = function(callback) { + var assetRegistryReady = false, + cacheRegistryReady = false; + + var onReady = function() { + var assetDict = JSON.parse(JSON.stringify(assetSourceRegistry)), + cacheDict = assetCacheRegistry, + assetEntry, cacheEntry, + now = Date.now(), obsoleteAfter; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry ) { + assetEntry.cached = true; + assetEntry.writeTime = cacheEntry.writeTime; + obsoleteAfter = cacheEntry.writeTime + assetEntry.updateAfter * 86400000; + assetEntry.obsolete = obsoleteAfter < now; + assetEntry.remoteURL = cacheEntry.remoteURL; } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + assetEntry.writeTime = 0; + obsoleteAfter = 0; + assetEntry.obsolete = true; } - return; } - //console.log('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s")', path, homeURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); + callback(assetDict); }; - var onHomeFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', homeURL)); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); - } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - } - }; - - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Auto-update enabled AND (not in cache OR in cache but obsolete) - var timestamp = entries[path]; - var inCache = typeof timestamp === 'number'; - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && stringIsNotEmpty(homeURL) - ) { - if ( inCache === false || cacheIsObsolete(timestamp) ) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheMetaReady(): not cached or obsolete', path); - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - return; - } - } - - // In cache - if ( inCache ) { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - - // Not in cache - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); - }; - - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; - } - - // Repo copy changed: fetch from home URL - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µBlock> readRepoCopyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - if ( stringIsNotEmpty(homeURL) ) { - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - } else { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); - } - return; - } - - // Load from cache - cachedAssetsManager.entries(onCacheMetaReady); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -// https://www.youtube.com/watch?v=uvUW4ozs7pY - -/******************************************************************************/ - -// An important asset shipped with the extension -- typically small, or -// doesn't change often: -// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL -// Repository --> has checksum (to detect need for update and corruption) -// Cache --> whatever from above -// Local --> install time version - -var readRepoOnlyAsset = function(path, callback) { - - var assetEntry; - - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } - callback(details); - }; - - var onInstallFileLoaded = function() { - //console.log('µBlock> readRepoOnlyAsset("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µBlock> readRepoOnlyAsset("%s") / onInstallFileError()', path); - reportBack('', 'Error'); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentError()', path); - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); - }; - - var repositoryURL = toRepoURL(path + '?_=' + Date.now()); - - var onRepoFileLoaded = function() { - if ( typeof this.responseText !== 'string' ) { - console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): no response', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - if ( YaMD5.hashStr(this.responseText) !== assetEntry.repoChecksum ) { - console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): bad md5 checksum', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; - } - - // Asset added or changed: load from repo URL and then cache result - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - getTextFileFromURL(repositoryURL, onRepoFileLoaded, onRepoFileError); - return; - } - - // Load from cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -/******************************************************************************/ - -// Asset doesn't exist. Just for symmetry purpose. - -var readNilAsset = function(path, callback) { - callback({ - 'path': path, - 'content': '', - 'error': 'Error: asset not found' + getAssetSourceRegistry(function() { + assetRegistryReady = true; + if ( cacheRegistryReady ) { onReady(); } }); -}; - -/******************************************************************************/ -// An external asset: -// Path --> starts with 'http' -// External --> https://..., http://... -// Cache --> has expiration timestamp (in cache) - -var readExternalAsset = function(path, callback) { - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } - callback(details); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readExternalAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - console.error('µBlock> readExternalAsset("%s") / onCachedContentError()', path); - reportBack('', 'Error'); - }; - - var onExternalFileLoaded = function() { - // https://github.com/chrisaljoudi/uBlock/issues/708 - // A successful download should never return an empty file: turn this - // into an error condition. - if ( stringIsNotEmpty(this.responseText) === false ) { - onExternalFileError(); - return; - } - //console.log('µBlock> readExternalAsset("%s") / onExternalFileLoaded1()', path); - cachedAssetsManager.save(path, this.responseText); - reportBack(this.responseText); - }; - - var onExternalFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', path)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Not in cache OR - // - // - Auto-update enabled AND in cache but obsolete - var timestamp = entries[path]; - var notInCache = typeof timestamp !== 'number'; - var updateCache = exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - cacheIsObsolete(timestamp); - if ( notInCache || updateCache ) { - getTextFileFromURL(path, onExternalFileLoaded, onExternalFileError); - return; - } - - // In cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - cachedAssetsManager.entries(onCacheMetaReady); + getAssetCacheRegistry(function() { + cacheRegistryReady = assetCacheRegistry; + if ( assetRegistryReady ) { onReady(); } + }); }; /******************************************************************************/ -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved - -var readUserAsset = function(path, callback) { - // TODO: remove when confident all users no longer have their custom - // filters saved into vAPI.cacheStorage. - var onCachedContentLoaded = function(details) { - saveUserAsset(path, details.content); - //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); - }; - - var onCachedContentError = function() { - saveUserAsset(path, ''); - //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); - }; - - var onLoaded = function(bin) { - var content = bin && bin[path]; - if ( typeof content === 'string' ) { - callback({ 'path': path, 'content': content }); - return; - } - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - vAPI.storage.get(path, onLoaded); +api.purge = function(pattern, callback) { + assetCacheMarkAsDirty(pattern, callback); }; -var saveUserAsset = function(path, content, callback) { - var bin = {}; - bin[path] = content; - var onSaved = function() { - // Saving over an existing asset must be seen as removing an - // existing asset and adding a new one. - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener([ path ]); - } - if ( callback instanceof Function ) { - callback({ path: path, content: content }); - } - }; - vAPI.storage.set(bin, onSaved); +api.remove = function(pattern, callback) { + assetCacheRemove(pattern, callback); }; -/******************************************************************************/ - -// Asset available only from the cache. -// Cache data: -// Path --> starts with 'cache://' -// Cache --> whatever - -var readCacheAsset = function(path, callback) { - var onCachedContentLoaded = function(details) { - //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); - }; - - var onCachedContentError = function() { - //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); - }; - - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); +api.rmrf = function() { + assetCacheRemove(/./); }; /******************************************************************************/ -// Assets -// -// A copy of an asset from an external source shipped with the extension: -// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL -// External --> -// Repository --> has checksum (to detect obsolescence) -// Cache --> has expiration timestamp (to detect obsolescence) -// Local --> install time version -// -// An important asset shipped with the extension (usually small, or doesn't -// change often): -// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL -// Repository --> has checksum (to detect obsolescence or data corruption) -// Cache --> whatever from above -// Local --> install time version -// -// An external filter list: -// Path --> starts with 'http' -// External --> -// Cache --> has expiration timestamp (to detect obsolescence) -// -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved -// -// When a checksum is present, it is used to determine whether the asset -// needs to be updated. -// When an expiration timestamp is present, it is used to determine whether -// the asset needs to be updated. -// -// If no update required, an asset if first fetched from the cache. If the -// asset is not cached it is fetched from the closest location: local for -// an asset shipped with the extension, external for an asset not shipped -// with the extension. - -exports.get = function(path, callback) { - - if ( reIsUserPath.test(path) ) { - readUserAsset(path, callback); - return; - } +// Asset updater area. - if ( reIsCachePath.test(path) ) { - readCacheAsset(path, callback); - return; - } +var updateStatus; +var updateTimer; +var updateAssetDelay = 4000; +var updated = []; - if ( reIsExternalPath.test(path) ) { - readExternalAsset(path, callback); - return; - } +var updateFirst = function() { + updateStatus = 'updating'; + updated = []; - var onRepoMetaReady = function(meta) { - var assetEntry = meta.entries[path]; + fireNotification('before-assets-updated'); - // Asset doesn't exist - if ( assetEntry === undefined ) { - readNilAsset(path, callback); - return; - } - - // Asset is repo copy of external content - if ( stringIsNotEmpty(homeURLs[path]) ) { - readRepoCopyAsset(path, callback); - return; - } - - // Asset is repo only - readRepoOnlyAsset(path, callback); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -// https://www.youtube.com/watch?v=98y0Q7nLGWk - -/******************************************************************************/ - -exports.getLocal = readLocalFile; - -/******************************************************************************/ - -exports.put = function(path, content, callback) { - if ( reIsUserPath.test(path) ) { - saveUserAsset(path, content, callback); - return; - } - - cachedAssetsManager.save(path, content, callback); -}; - -/******************************************************************************/ - -exports.rmrf = function() { - cachedAssetsManager.rmrf(); + updateNext(); }; -/******************************************************************************/ +var updateNext = function() { -exports.rename = function(from, to, callback) { - var done = function() { - if ( typeof callback === 'function' ) { - callback(); - } - }; - - var fromLoaded = function(details) { - cachedAssetsManager.remove(from); - cachedAssetsManager.save(to, details.content, callback); - done(); - }; + var assetDict, cacheDict; - var toLoaded = function(details) { - // `to` already exists: do nothing - if ( details.content !== '' ) { - return done(); - } - cachedAssetsManager.load(from, fromLoaded); - }; - - // If `to` content already exists, do nothing. - cachedAssetsManager.load(to, toLoaded); -}; - -/******************************************************************************/ - -exports.metadata = function(callback) { - var out = {}; - - // https://github.com/chrisaljoudi/uBlock/issues/186 - // We need to check cache obsolescence when both cache and repo meta data - // has been gathered. - var checkCacheObsolescence = function() { - var entry, homeURL; - for ( var path in out ) { - if ( out.hasOwnProperty(path) === false ) { + var findOne = function() { + var now = Date.now(), + assetEntry, cacheEntry; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + if ( assetEntry.hasRemoteURL !== true ) { continue; } + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { continue; } - entry = out[path]; - // https://github.com/gorhill/uBlock/issues/528 - // Not having a homeURL property does not mean the filter list - // is not external. - homeURL = reIsExternalPath.test(path) ? path : homeURLs[path]; - entry.cacheObsolete = stringIsNotEmpty(homeURL) && - cacheIsObsolete(entry.lastModified); - } - callback(out); - }; - - var onRepoMetaReady = function(meta) { - var entries = meta.entries; - var entryRepo, entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entryRepo = entries[path]; - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; - } - entryOut.localChecksum = entryRepo.localChecksum; - entryOut.repoChecksum = entryRepo.repoChecksum; - entryOut.homeURL = homeURLs[path] || ''; - entryOut.supportURL = entryRepo.supportURL || ''; - entryOut.repoObsolete = entryOut.localChecksum !== entryOut.repoChecksum; - } - checkCacheObsolescence(); - }; - - var onCacheMetaReady = function(entries) { - var entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; - } - entryOut.lastModified = entries[path]; - // User data is not literally cache data - if ( reIsUserPath.test(path) ) { - continue; + if ( assetEntry.error ) { + if ( assetEntry.error.time + 3600000 > now ) { + continue; + } } - entryOut.cached = true; - if ( reIsExternalPath.test(path) ) { - entryOut.homeURL = path; + if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) { + return assetKey; } } - getRepoMetadata(onRepoMetaReady); }; - cachedAssetsManager.entries(onCacheMetaReady); -}; - -/******************************************************************************/ - -exports.purge = function(pattern, before) { - cachedAssetsManager.remove(pattern, before); -}; - -exports.purgeCacheableAsset = function(pattern, before) { - cachedAssetsManager.remove(pattern, before); - lastRepoMetaTimestamp = 0; -}; - -exports.purgeAll = function(callback) { - cachedAssetsManager.removeAll(callback); - lastRepoMetaTimestamp = 0; -}; - -/******************************************************************************/ - -exports.onAssetRemoved = { - addListener: function(callback) { - onAssetRemovedListener = callback instanceof Function ? callback : null; - } -}; - -/******************************************************************************/ - -return exports; - -})(); - -/******************************************************************************/ -/******************************************************************************/ - -µBlock.assetUpdater = (function() { - -/******************************************************************************/ - -var µb = µBlock; - -var updateDaemonTimer = null; -var autoUpdateDaemonTimerPeriod = 11 * 60 * 1000; // 11 minutes -var manualUpdateDaemonTimerPeriod = 5 * 1000; // 5 seconds - -var updateCycleFirstPeriod = 7 * 60 * 1000; // 7 minutes -var updateCycleNextPeriod = 11 * 60 * 60 * 1000; // 11 hours -var updateCycleTime = 0; - -var toUpdate = {}; -var toUpdateCount = 0; -var updated = {}; -var updatedCount = 0; -var metadata = null; - -var onStartListener = null; -var onCompletedListener = null; -var onAssetUpdatedListener = null; - -var exports = { - manualUpdate: false, - manualUpdateProgress: { - value: 0, - text: null - } -}; - -/******************************************************************************/ - -var onOneUpdated = function(details) { - // Resource fetched, we can safely restart the daemon. - scheduleUpdateDaemon(); - - var path = details.path; - if ( details.error ) { - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount)); - //console.debug('µBlock.assetUpdater/onOneUpdated: "%s" failed', path); - return; - } - - //console.debug('µBlock.assetUpdater/onOneUpdated: "%s"', path); - updated[path] = true; - updatedCount += 1; - - if ( typeof onAssetUpdatedListener === 'function' ) { - onAssetUpdatedListener(details); - } - - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount + 1)); -}; - -/******************************************************************************/ - -var updateOne = function() { - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var metaEntry; - var updatingCount = 0; - var updatingText = null; - - for ( var path in toUpdate ) { - if ( toUpdate.hasOwnProperty(path) === false ) { - continue; - } - if ( toUpdate[path] !== true ) { - continue; - } - toUpdate[path] = false; - toUpdateCount -= 1; - if ( metadata.hasOwnProperty(path) === false ) { - continue; + var updatedOne = function(details) { + if ( details.content !== '' ) { + updated.push(details.assetKey); } - metaEntry = metadata[path]; - if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) { - continue; + if ( findOne() !== undefined ) { + vAPI.setTimeout(updateNext, updateAssetDelay); + } else { + updateDone(); } - - // Will restart the update daemon once the resource is received: the - // fetching of a resource may take some time, possibly beyond the - // next scheduled daemon cycle, so this ensure the daemon won't do - // anything else before the resource is fetched (or times out). - suspendUpdateDaemon(); - - //console.debug('µBlock.assetUpdater/updateOne: assets.get("%s")', path); - µb.assets.get(path, onOneUpdated); - updatingCount = 1; - updatingText = metaEntry.homeURL || path; - break; - } - - manualUpdateNotify( - false, - (updatedCount + updatingCount/2) / (updatedCount + toUpdateCount + updatingCount + 1), - updatingText - ); -}; - -/******************************************************************************/ - -// Update one asset, fetch metadata if not done yet. - -var safeUpdateOne = function() { - if ( metadata !== null ) { - updateOne(); - return; - } - - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var onMetadataReady = function(response) { - scheduleUpdateDaemon(); - metadata = response; - updateOne(); }; - suspendUpdateDaemon(); - µb.assets.metadata(onMetadataReady); -}; - -/******************************************************************************/ - -var safeStartListener = function(callback) { - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var onStartListenerDone = function(assets) { - scheduleUpdateDaemon(); - assets = assets || {}; - for ( var path in assets ) { - if ( assets.hasOwnProperty(path) === false ) { - continue; - } - if ( toUpdate.hasOwnProperty(path) ) { - continue; - } - //console.debug('assets.js > µBlock.assetUpdater/safeStartListener: "%s"', path); - toUpdate[path] = true; - toUpdateCount += 1; - } - if ( typeof callback === 'function' ) { - callback(); + var updateOne = function() { + var assetKey = findOne(); + if ( assetKey === undefined ) { + return updateDone(); } + getRemote(assetKey, updatedOne); }; - if ( typeof onStartListener === 'function' ) { - suspendUpdateDaemon(); - onStartListener(onStartListenerDone); - } else { - onStartListenerDone(null); - } -}; - -/******************************************************************************/ - -var updateDaemon = function() { - updateDaemonTimer = null; - scheduleUpdateDaemon(); - - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - if ( µb.assets.autoUpdate !== true ) { - return; - } - - // Start an update cycle? - if ( updateCycleTime !== 0 ) { - if ( Date.now() >= updateCycleTime ) { - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle started'); - reset(); - safeStartListener(); - } - return; - } - - // Any asset to update? - if ( toUpdateCount !== 0 ) { - safeUpdateOne(); - return; - } - // Nothing left to update - - // In case of manual update, fire progress notifications - manualUpdateNotify(true, 1, ''); - - // If anything was updated, notify listener - if ( updatedCount !== 0 ) { - if ( typeof onCompletedListener === 'function' ) { - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle completed'); - onCompletedListener({ - updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy - updatedCount: updatedCount - }); - } - } - - // Schedule next update cycle - if ( updateCycleTime === 0 ) { - reset(); - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle re-scheduled'); - updateCycleTime = Date.now() + updateCycleNextPeriod; - } -}; - -/******************************************************************************/ - -var scheduleUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - } - updateDaemonTimer = vAPI.setTimeout( - updateDaemon, - exports.manualUpdate ? manualUpdateDaemonTimerPeriod : autoUpdateDaemonTimerPeriod - ); -}; - -var suspendUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - updateDaemonTimer = null; - } -}; - -scheduleUpdateDaemon(); - -/******************************************************************************/ - -var reset = function() { - toUpdate = {}; - toUpdateCount = 0; - updated = {}; - updatedCount = 0; - updateCycleTime = 0; - metadata = null; -}; - -/******************************************************************************/ - -var manualUpdateNotify = function(done, value, text) { - if ( exports.manualUpdate === false ) { - return; - } - - exports.manualUpdate = !done; - exports.manualUpdateProgress.value = value || 0; - if ( typeof text === 'string' ) { - exports.manualUpdateProgress.text = text; - } - - vAPI.messaging.broadcast({ - what: 'forceUpdateAssetsProgress', - done: !exports.manualUpdate, - progress: exports.manualUpdateProgress, - updatedCount: updatedCount + getAssetSourceRegistry(function(dict) { + assetDict = dict; + if ( !cacheDict ) { return; } + updateOne(); }); - // When manually updating, whatever launched the manual update is - // responsible to launch a reload of the filter lists. - if ( exports.manualUpdate !== true ) { - reset(); - } + getAssetCacheRegistry(function(dict) { + cacheDict = dict; + if ( !assetDict ) { return; } + updateOne(); + }); }; -/******************************************************************************/ - -// Manual update: just a matter of forcing the update daemon to work on a -// tighter schedule. - -exports.force = function() { - if ( exports.manualUpdate ) { - return; - } - - reset(); - - exports.manualUpdate = true; - - var onStartListenerDone = function() { - if ( toUpdateCount === 0 ) { - updateCycleTime = Date.now() + updateCycleNextPeriod; - manualUpdateNotify(true, 1); - } else { - manualUpdateNotify(false, 0); - safeUpdateOne(); - } - }; - - safeStartListener(onStartListenerDone); +var updateDone = function() { + var assetKeys = updated.slice(0); + updated = []; + updateStatus = undefined; + fireNotification('after-assets-updated', { assetKeys: assetKeys }); }; -/******************************************************************************/ - -exports.onStart = { - addEventListener: function(callback) { - onStartListener = callback || null; - if ( typeof onStartListener === 'function' ) { - updateCycleTime = Date.now() + updateCycleFirstPeriod; - } +api.updateStart = function(details) { + updateAssetDelay = details.delay; + if ( updateStatus !== undefined ) { + clearTimeout(updateTimer); + updateTimer = vAPI.setTimeout(updateNext, updateAssetDelay); + return; } + updateFirst(); }; -/******************************************************************************/ - -exports.onAssetUpdated = { - addEventListener: function(callback) { - onAssetUpdatedListener = callback || null; +api.updateStop = function() { + if ( updateTimer ) { + clearTimeout(updateTimer); + updateTimer = undefined; } -}; - -/******************************************************************************/ - -exports.onCompleted = { - addEventListener: function(callback) { - onCompletedListener = callback || null; + if ( updateStatus !== undefined ) { + updateDone(); } }; /******************************************************************************/ -// Typically called when an update has been forced. - -exports.restart = function() { - reset(); - updateCycleTime = Date.now() + updateCycleNextPeriod; -}; - -/******************************************************************************/ - -// Call when disabling uBlock, to ensure it doesn't stick around as a detached -// window object in Firefox. - -exports.shutdown = function() { - suspendUpdateDaemon(); - reset(); -}; +return api; /******************************************************************************/ -return exports; - })(); /******************************************************************************/ diff --git a/src/js/background.js b/src/js/background.js index 00b82a1c64997..bb7d462120473 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,13 +19,11 @@ Home: https://github.com/gorhill/uBlock */ -/* exported µBlock */ - 'use strict'; /******************************************************************************/ -var µBlock = (function() { +var µBlock = (function() { // jshint ignore:line /******************************************************************************/ @@ -123,88 +121,11 @@ return { // as per list headers. updateAssetsEvery: 97 * oneHour, projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/', - userFiltersPath: 'assets/user/filters.txt', - pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat', - - // permanent lists - permanentLists: { - // User - 'assets/user/filters.txt': { - group: 'default' - }, - // uBlock - 'assets/ublock/filters.txt': { - title: 'uBlock filters', - group: 'default' - }, - 'assets/ublock/privacy.txt': { - title: 'uBlock filters – Privacy', - group: 'default' - }, - 'assets/ublock/unbreak.txt': { - title: 'uBlock filters – Unbreak', - group: 'default' - }, - 'assets/ublock/badware.txt': { - title: 'uBlock filters – Badware risks', - group: 'default', - supportURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks', - instructionURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks' - }, - 'assets/ublock/experimental.txt': { - title: 'uBlock filters – Experimental', - group: 'default', - off: true, - supportURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters', - instructionURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters' - } - }, + userFiltersPath: 'user-filters', + pslAssetKey: 'public_suffix_list.dat', // current lists - remoteBlacklists: {}, - oldListToNewListMap: { - "assets/thirdparties/adblock.gardar.net/is.abp.txt": "http://adblock.gardar.net/is.abp.txt", - "assets/thirdparties/adblock.schack.dk/block.txt": "https://adblock.dk/block.csv", - "https://adblock.schack.dk/block.txt": "https://adblock.dk/block.csv", - "assets/thirdparties/dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/advblock.txt": "https://easylist-downloads.adblockplus.org/advblock.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/bitblock.txt": "https://easylist-downloads.adblockplus.org/bitblock.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistchina.txt": "https://easylist-downloads.adblockplus.org/easylistchina.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistdutch.txt": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistgermany.txt": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistitaly.txt": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-social.txt": "https://easylist-downloads.adblockplus.org/fanboy-social.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/liste_fr.txt": "https://easylist-downloads.adblockplus.org/liste_fr.txt", - "assets/thirdparties/gitorious.org/adblock-latvian/adblock-latvian/raw/master_lists/latvian-list.txt": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", - "assets/thirdparties/home.fredfiber.no/langsholt/adblock.txt": "http://home.fredfiber.no/langsholt/adblock.txt", - "assets/thirdparties/hosts-file.net/ad-servers": "http://hosts-file.net/.%5Cad_servers.txt", - "assets/thirdparties/http://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", - "assets/thirdparties/liste-ar-adblock.googlecode.com/hg/Liste_AR.txt": "https://liste-ar-adblock.googlecode.com/hg/Liste_AR.txt", - "assets/thirdparties/margevicius.lt/easylistlithuania.txt": "http://margevicius.lt/easylistlithuania.txt", - "assets/thirdparties/mirror1.malwaredomains.com/files/immortal_domains.txt": "http://malwaredomains.lehigh.edu/files/immortal_domains.txt", - "assets/thirdparties/raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt": "https://raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt", - "assets/thirdparties/raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", - "assets/thirdparties/raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", - "assets/thirdparties/raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", - "assets/thirdparties/raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", - "assets/thirdparties/someonewhocares.org/hosts/hosts": "http://someonewhocares.org/hosts/hosts", - "assets/thirdparties/spam404bl.com/spam404scamlist.txt": "https://spam404bl.com/spam404scamlist.txt", - "assets/thirdparties/stanev.org/abp/adblock_bg.txt": "http://stanev.org/abp/adblock_bg.txt", - "assets/thirdparties/winhelp2002.mvps.org/hosts.txt": "http://winhelp2002.mvps.org/hosts.txt", - "assets/thirdparties/www.fanboy.co.nz/enhancedstats.txt": "https://www.fanboy.co.nz/enhancedstats.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-antifacebook.txt": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-korean.txt": "https://www.fanboy.co.nz/fanboy-korean.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-swedish.txt": "https://www.fanboy.co.nz/fanboy-swedish.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-ultimate.txt": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-vietnam.txt": "https://www.fanboy.co.nz/fanboy-vietnam.txt", - "assets/thirdparties/www.void.gr/kargig/void-gr-filters.txt": "https://www.void.gr/kargig/void-gr-filters.txt", - "assets/thirdparties/www.zoso.ro/pages/rolist.txt": "", - "https://iadb.azurewebsites.net/Finland_adb.txt": "http://adb.juvander.net/Finland_adb.txt", - "https://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", - "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt" - }, + availableFilterLists: {}, selfieAfter: 23 * oneMinute, diff --git a/src/js/logger.js b/src/js/logger.js index b9a7bee328748..700521ab5eada 100644 --- a/src/js/logger.js +++ b/src/js/logger.js @@ -19,15 +19,13 @@ Home: https://github.com/gorhill/uBlock */ -/* global µBlock */ +'use strict'; /******************************************************************************/ /******************************************************************************/ µBlock.logger = (function() { -'use strict'; - /******************************************************************************/ /******************************************************************************/ diff --git a/src/js/messaging.js b/src/js/messaging.js index 360fc7e0f39fd..f71444b2f14b8 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -77,7 +77,7 @@ var onMessage = function(request, sender, callback) { return; case 'reloadAllFilters': - µb.reloadAllFilters(callback); + µb.loadFilterLists(); return; case 'scriptlet': @@ -121,7 +121,8 @@ var onMessage = function(request, sender, callback) { break; case 'forceUpdateAssets': - µb.assetUpdater.force(); + µb.scheduleAssetUpdater(0); + µb.assets.updateStart({ delay: request.fast ? 1 : 3 * 1000 }); break; case 'getAppData': @@ -160,7 +161,7 @@ var onMessage = function(request, sender, callback) { break; case 'selectFilterLists': - µb.selectFilterLists(request.switches); + µb.saveSelectedFilterLists(request.keys, request.append); break; case 'setWhitelist': @@ -753,7 +754,7 @@ var backupUserData = function(callback) { timeStamp: Date.now(), version: vAPI.app.version, userSettings: µb.userSettings, - filterLists: {}, + selectedFilterLists: [], hiddenSettingsString: µb.stringFromHiddenSettings(), netWhitelist: µb.stringFromWhitelist(µb.netWhitelist), dynamicFilteringString: µb.permanentFirewall.toString(), @@ -762,8 +763,8 @@ var backupUserData = function(callback) { userFilters: '' }; - var onSelectedListsReady = function(filterLists) { - userData.filterLists = filterLists; + var onSelectedListsReady = function(selectedFilterLists) { + userData.selectedFilterLists = selectedFilterLists; var filename = vAPI.i18n('aboutBackupFilename') .replace('{{datetime}}', µb.dateNowToSensibleString()) @@ -783,7 +784,7 @@ var backupUserData = function(callback) { var onUserFiltersReady = function(details) { userData.userFilters = details.content; - µb.extractSelectedFilterLists(onSelectedListsReady); + µb.loadSelectedFilterLists(onSelectedListsReady); }; µb.assets.get(µb.userFiltersPath, onUserFiltersReady); @@ -804,7 +805,7 @@ var restoreUserData = function(request) { µb.keyvalSetOne('version', userData.version); µBlock.saveLocalSettings(); vAPI.storage.set(userData.userSettings, onCountdown); - µb.keyvalSetOne('remoteBlacklists', userData.filterLists, onCountdown); + µb.keyvalSetOne('filterLists', userData.filterLists, onCountdown); µb.hiddenSettingsFromString(userData.hiddenSettingsString || ''); µb.keyvalSetOne('netWhitelist', userData.netWhitelist || '', onCountdown); µb.keyvalSetOne('dynamicFilteringString', userData.dynamicFilteringString || '', onCountdown); @@ -869,16 +870,13 @@ var getLists = function(callback) { cache: null, parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters, cosmeticFilterCount: µb.cosmeticFilteringEngine.getFilterCount(), - current: µb.remoteBlacklists, + current: µb.availableFilterLists, ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters, - manualUpdate: false, netFilterCount: µb.staticNetFilteringEngine.getFilterCount(), userFiltersPath: µb.userFiltersPath }; var onMetadataReady = function(entries) { r.cache = entries; - r.manualUpdate = µb.assetUpdater.manualUpdate; - r.manualUpdateProgress = µb.assetUpdater.manualUpdateProgress; prepListEntries(r.cache); callback(r); }; @@ -953,7 +951,10 @@ var onMessage = function(request, sender, callback) { return getLocalData(callback); case 'purgeAllCaches': - return µb.assets.purgeAll(callback); + if ( request.hard ) { + return µb.assets.remove(/./, callback); + } + return µb.assets.purge(/./, callback); case 'readUserFilters': return µb.loadUserFilters(callback); @@ -974,7 +975,7 @@ var onMessage = function(request, sender, callback) { break; case 'purgeCache': - µb.assets.purgeCacheableAsset(request.path); + µb.assets.purge(request.path); break; case 'readHiddenSettings': diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 473bffa79a372..dfae64ffd158d 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -402,27 +402,15 @@ RedirectEngine.prototype.resourceContentFromName = function(name, mime) { // TODO: combine same key-redirect pairs into a single regex. RedirectEngine.prototype.resourcesFromString = function(text) { - var textEnd = text.length; - var lineBeg = 0, lineEnd; - var line, fields, encoded; - var reNonEmptyLine = /\S/; + var line, fields, encoded, + reNonEmptyLine = /\S/, + lineIter = new µBlock.LineIterator(text); this.resources = new Map(); - while ( lineBeg < textEnd ) { - lineEnd = text.indexOf('\n', lineBeg); - if ( lineEnd < 0 ) { - lineEnd = text.indexOf('\r', lineBeg); - if ( lineEnd < 0 ) { - lineEnd = textEnd; - } - } - line = text.slice(lineBeg, lineEnd); - lineBeg = lineEnd + 1; - - if ( line.startsWith('#') ) { - continue; - } + while ( lineIter.eot() === false ) { + line = lineIter.next(); + if ( line.startsWith('#') ) { continue; } if ( fields === undefined ) { fields = line.trim().split(/\s+/); diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 79af14e7c571e..f2a5961121a0d 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -1,7 +1,7 @@ /******************************************************************************* - uBlock - a browser extension to block requests. - Copyright (C) 2015 Raymond Hill + uBlock Origin - a browser extension to block requests. + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,14 +19,12 @@ Home: https://github.com/gorhill/uBlock */ -/* global µBlock */ +'use strict'; /******************************************************************************/ µBlock.staticFilteringReverseLookup = (function() { -'use strict'; - /******************************************************************************/ var worker = null; @@ -99,18 +97,18 @@ var initWorker = function(callback) { }; var µb = µBlock; - var path, entry; + var listKey, entry; - for ( path in µb.remoteBlacklists ) { - if ( µb.remoteBlacklists.hasOwnProperty(path) === false ) { - continue; - } - entry = µb.remoteBlacklists[path]; - if ( entry.off === true ) { + for ( listKey in µb.availableFilterLists ) { + if ( µb.availableFilterLists.hasOwnProperty(listKey) === false ) { continue; } - entries[path] = { - title: path !== µb.userFiltersPath ? entry.title : vAPI.i18n('1pPageName'), + entry = µb.availableFilterLists[listKey]; + if ( entry.off === true ) { continue; } + entries[listKey] = { + title: listKey !== µb.userFiltersPath ? + entry.title : + vAPI.i18n('1pPageName'), supportURL: entry.supportURL || '' }; countdown += 1; @@ -121,8 +119,8 @@ var initWorker = function(callback) { return; } - for ( path in entries ) { - µb.getCompiledFilterList(path, onListLoaded); + for ( listKey in entries ) { + µb.getCompiledFilterList(listKey, onListLoaded); } }; diff --git a/src/js/scriptlets/subscriber.js b/src/js/scriptlets/subscriber.js index 9c525de0d9b36..887e95db831f3 100644 --- a/src/js/scriptlets/subscriber.js +++ b/src/js/scriptlets/subscriber.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2015-2016 Raymond Hill + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,6 +21,8 @@ /* global vAPI, HTMLDocument */ +'use strict'; + /******************************************************************************/ // Injected into specific web pages, those which have been pre-selected @@ -30,8 +32,6 @@ (function() { -'use strict'; - /******************************************************************************/ // https://github.com/chrisaljoudi/uBlock/issues/464 @@ -100,7 +100,8 @@ var onAbpLinkClicked = function(ev) { 'scriptlets', { what: 'selectFilterLists', - switches: [ { location: location, off: false } ] + keys: [ location ], + append: true }, onListsSelectionDone ); diff --git a/src/js/start.js b/src/js/start.js index 12a583db48137..97a80408310bf 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -39,7 +39,7 @@ var µb = µBlock; vAPI.app.onShutdown = function() { µb.staticFilteringReverseLookup.shutdown(); - µb.assetUpdater.shutdown(); + µb.assets.updateStop(); µb.staticNetFilteringEngine.reset(); µb.cosmeticFilteringEngine.reset(); µb.sessionFirewall.reset(); @@ -58,14 +58,8 @@ vAPI.app.onShutdown = function() { var onAllReady = function() { // https://github.com/chrisaljoudi/uBlock/issues/184 // Check for updates not too far in the future. - µb.assetUpdater.onStart.addEventListener(µb.updateStartHandler.bind(µb)); - µb.assetUpdater.onCompleted.addEventListener(µb.updateCompleteHandler.bind(µb)); - µb.assetUpdater.onAssetUpdated.addEventListener(µb.assetUpdatedHandler.bind(µb)); - µb.assets.onAssetRemoved.addListener(µb.assetCacheRemovedHandler.bind(µb)); - - // Important: remove barrier to remote fetching, this was useful only - // for launch time. - µb.assets.remoteFetchBarrier -= 1; + µb.assets.addObserver(µb.assetObserver.bind(µb)); + µb.scheduleAssetUpdater(µb.userSettings.autoUpdate ? 7 * 60 * 1000 : 0); // vAPI.cloud is optional. if ( µb.cloudStorageSupported ) { @@ -129,7 +123,7 @@ var onSelfieReady = function(selfie) { return false; } - µb.remoteBlacklists = selfie.filterLists; + µb.availableFilterLists = selfie.availableFilterLists; µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); µb.redirectEngine.fromSelfie(selfie.redirectEngine); µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine); @@ -157,12 +151,6 @@ var onUserSettingsReady = function(fetched) { fromFetch(userSettings, fetched); - // https://github.com/chrisaljoudi/uBlock/issues/426 - // Important: block remote fetching for when loading assets at launch - // time. - µb.assets.autoUpdate = userSettings.autoUpdate; - µb.assets.autoUpdateDelay = µb.updateAssetsEvery; - if ( µb.privacySettingsSupported ) { vAPI.browserSettings.set({ 'hyperlinkAuditing': !userSettings.hyperlinkAuditingDisabled, @@ -192,7 +180,7 @@ var onUserSettingsReady = function(fetched) { var onSystemSettingsReady = function(fetched) { var mustSaveSystemSettings = false; if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) { - µb.assets.purge(/^cache:\/\/compiled-/); + µb.assets.remove(/^compiled\//); mustSaveSystemSettings = true; } if ( fetched.selfieMagic !== µb.systemSettings.selfieMagic ) { @@ -254,9 +242,6 @@ var fromFetch = function(to, fetched) { /******************************************************************************/ var onAdminSettingsRestored = function() { - // Forbid remote fetching of assets - µb.assets.remoteFetchBarrier += 1; - var fetchableProps = { 'compiledMagic': '', 'dynamicFilteringString': 'behind-the-scene * 3p noop\nbehind-the-scene * 3p-frame noop', diff --git a/src/js/storage.js b/src/js/storage.js index db079f5ac1b8f..9e29ec89ab9f1 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uBlock */ -/* global YaMD5, objectAssign, punycode, publicSuffixList */ +/* global objectAssign, punycode, publicSuffixList */ 'use strict'; @@ -153,52 +153,20 @@ /******************************************************************************/ -// This will remove all unused filter list entries from -// µBlock.remoteBlacklists`. This helps reduce the size of backup files. - -µBlock.extractSelectedFilterLists = function(callback) { - var µb = this; - - var onBuiltinListsLoaded = function(details) { - var builtin; - try { - builtin = JSON.parse(details.content); - } catch (e) { - builtin = {}; - } - - var result = JSON.parse(JSON.stringify(µb.remoteBlacklists)); - var entry, builtinPath, defaultState; - - for ( var path in result ) { - if ( result.hasOwnProperty(path) === false ) { - continue; - } - entry = result[path]; - // https://github.com/gorhill/uBlock/issues/277 - // uBlock's filter lists are always enabled by default, so we - // have to include in backup only those which are turned off. - if ( path.startsWith('assets/ublock/') ) { - if ( entry.off !== true ) { - delete result[path]; - } - continue; - } - builtinPath = path.replace(/^assets\/thirdparties\//, ''); - defaultState = builtin.hasOwnProperty(builtinPath) === false || - builtin[builtinPath].off === true; - if ( entry.off === true && entry.off === defaultState ) { - delete result[path]; - } - } - - callback(result); - }; +µBlock.loadSelectedFilterLists = function(callback) { + vAPI.storage.get('selectedFilterLists', function(bin) { + callback(bin && bin.selectedFilterLists || undefined); + }); +}; - // https://github.com/gorhill/uBlock/issues/63 - // Get built-in block lists: this will help us determine whether a - // specific list must be included in the result. - this.loadAndPatchStockFilterLists(onBuiltinListsLoaded); +µBlock.saveSelectedFilterLists = function(listKeys, append) { + if ( append ) { + this.loadSelectedFilterLists(function(keys) { + vAPI.storage.set({ selectedFilterLists: listKeys.concat(keys || []) }); + }); + } else { + vAPI.storage.set({ selectedFilterLists: listKeys }); + } }; /******************************************************************************/ @@ -207,14 +175,11 @@ // https://github.com/gorhill/uBlock/issues/1022 // Be sure to end with an empty line. content = content.trim(); - if ( content !== '' ) { - content += '\n'; - } + if ( content !== '' ) { content += '\n'; } this.assets.put(this.userFiltersPath, content, callback); + this.removeCompiledFilterList(this.userFiltersPath); }; -/******************************************************************************/ - µBlock.loadUserFilters = function(callback) { return this.assets.get(this.userFiltersPath, callback); }; @@ -229,18 +194,18 @@ var µb = this; var onSaved = function() { - var compiledFilters = µb.compileFilters(filters); - var snfe = µb.staticNetFilteringEngine; - var cfe = µb.cosmeticFilteringEngine; - var acceptedCount = snfe.acceptedCount + cfe.acceptedCount; - var discardedCount = snfe.discardedCount + cfe.discardedCount; + var compiledFilters = µb.compileFilters(filters), + snfe = µb.staticNetFilteringEngine, + cfe = µb.cosmeticFilteringEngine, + acceptedCount = snfe.acceptedCount + cfe.acceptedCount, + discardedCount = snfe.discardedCount + cfe.discardedCount; µb.applyCompiledFilters(compiledFilters, true); - var entry = µb.remoteBlacklists[µb.userFiltersPath]; - var deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; - var deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); + var entry = µb.availableFilterLists[µb.userFiltersPath], + deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount, + deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); entry.entryCount += deltaEntryCount; entry.entryUsedCount += deltaEntryUsedCount; - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists }); µb.staticNetFilteringEngine.freeze(); µb.redirectEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); @@ -263,166 +228,187 @@ /******************************************************************************/ -µBlock.getAvailableLists = function(callback) { - var availableLists = {}; - var relocationMap = {}; - - var fixLocation = function(location) { - // https://github.com/chrisaljoudi/uBlock/issues/418 - // We now support built-in external filter lists - if ( /^https?:/.test(location) === false ) { - location = 'assets/thirdparties/' + location; +µBlock.listKeysFromCustomFilterLists = function(raw) { + var out = {}; + var reIgnore = /^[!#]|[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/, + lineIter = new this.LineIterator(raw), + location; + while ( lineIter.eot() === false ) { + location = lineIter.next().trim(); + if ( location === '' || reIgnore.test(location) ) { continue; } + out[location] = true; + } + return Object.keys(out); +}; + +/******************************************************************************/ + +µBlock.autoSelectRegionalFilterLists = function(lists) { + var lang = self.navigator.language.slice(0, 2), + selectedListKeys = [], + list; + for ( var key in lists ) { + if ( lists.hasOwnProperty(key) === false ) { continue; } + list = lists[key]; + if ( list.off !== true ) { + selectedListKeys.push(key); + continue; } - return location; - }; + if ( list.lang === lang ) { + selectedListKeys.push(key); + list.off = false; + } + } + return selectedListKeys; +}; - // selected lists - var onSelectedListsLoaded = function(store) { - var µb = µBlock; - var lists = store.remoteBlacklists; - var locations = Object.keys(lists); - var location, availableEntry, storedEntry; - var off; - - while ( (location = locations.pop()) ) { - storedEntry = lists[location]; - off = storedEntry.off === true; - // New location? - if ( relocationMap.hasOwnProperty(location) ) { - µb.purgeFilterList(location); - location = relocationMap[location]; - if ( off && lists.hasOwnProperty(location) ) { - off = lists[location].off === true; - } +/******************************************************************************/ + +µBlock.changeExternalFilterLists = function(before, after) { + var µb = µBlock; + var onLoaded = function(keys) { + var fullDict = new Set(keys || []), + mustSave = false, + oldKeys = µb.listKeysFromCustomFilterLists(before), + oldDict = new Set(oldKeys), + newKeys = µb.listKeysFromCustomFilterLists(after), + newDict = new Set(newKeys), + i, key; + i = oldKeys.length; + while ( i-- ) { + key = oldKeys[i]; + if ( fullDict.has(key) && !newDict.has(key) ) { + fullDict.delete(key); + mustSave = true; } - availableEntry = availableLists[location]; - if ( availableEntry === undefined ) { - µb.purgeFilterList(location); - continue; + } + i = newKeys.length; + while ( i-- ) { + key = newKeys[i]; + if ( !fullDict.has(key) && !oldDict.has(key) ) { + fullDict.add(key); + mustSave = true; } - availableEntry.off = off; - if ( typeof availableEntry.homeURL === 'string' ) { - µb.assets.setHomeURL(location, availableEntry.homeURL); + } + if ( mustSave ) { + µb.saveSelectedFilterLists(µb.setToArray(fullDict)); + } + }; + this.loadSelectedFilterLists(onLoaded); +}; + +/******************************************************************************/ + +µBlock.getAvailableLists = function(callback) { + var µb = this, + oldAvailableLists = {}, + newAvailableLists = {}; + + // User filter list. + newAvailableLists[this.userFiltersPath] = { + group: 'default', + title: vAPI.i18n('1pPageName') + }; + + // Custom filter lists. + var listKeys = this.listKeysFromCustomFilterLists(µb.userSettings.externalLists), + i = listKeys.length, listKey, entry; + while ( i-- ) { + listKey = listKeys[i]; + entry = { + content: 'filters', + contentURL: listKeys[i], + external: true, + group: 'custom', + submitter: 'user', + title: '' + }; + newAvailableLists[listKey] = entry; + this.assets.registerAssetSource(listKey, entry); + } + + // Reuse existing list metadata if any. + var reuseMetadata = function() { + var assetKeys = Object.keys(oldAvailableLists), + assetKey, newEntry, oldEntry; + + while ( (assetKey = assetKeys.pop()) ) { + oldEntry = oldAvailableLists[assetKey]; + newEntry = newAvailableLists[assetKey]; + if ( newEntry === undefined ) { + µb.removeFilterList(assetKey); + continue; } - if ( storedEntry.entryCount !== undefined ) { - availableEntry.entryCount = storedEntry.entryCount; + if ( oldEntry.entryCount !== undefined ) { + newEntry.entryCount = oldEntry.entryCount; } - if ( storedEntry.entryUsedCount !== undefined ) { - availableEntry.entryUsedCount = storedEntry.entryUsedCount; + if ( oldEntry.entryUsedCount !== undefined ) { + newEntry.entryUsedCount = oldEntry.entryUsedCount; } // This may happen if the list name was pulled from the list // content. // https://github.com/chrisaljoudi/uBlock/issues/982 // There is no guarantee the title was successfully extracted from // the list content. - if ( availableEntry.title === '' && - typeof storedEntry.title === 'string' && - storedEntry.title !== '' + if ( + newEntry.title === '' && + typeof oldEntry.title === 'string' && + oldEntry.title !== '' ) { - availableEntry.title = storedEntry.title; + newEntry.title = oldEntry.title; } } + }; - // https://github.com/gorhill/uBlock/issues/747 - if ( µb.firstInstall ) { - µb.autoSelectFilterLists(availableLists); + // Selected lists. + var onSelectedListsLoaded = function(keys) { + var listKey; + // No user lists data means use default settings. + if ( Array.isArray(keys) ) { + var listKeySet = new Set(keys); + for ( listKey in newAvailableLists ) { + if ( newAvailableLists.hasOwnProperty(listKey) ) { + newAvailableLists[listKey].off = !listKeySet.has(listKey); + } + } + } else if ( µb.firstInstall ) { + µb.saveSelectedFilterLists(µb.autoSelectRegionalFilterLists(newAvailableLists)); } - callback(availableLists); + reuseMetadata(); + callback(newAvailableLists); }; - // built-in lists + // Built-in filter lists. var onBuiltinListsLoaded = function(details) { - var location, locations; + var entries, entry; try { - locations = JSON.parse(details.content); + entries = JSON.parse(details.content); } catch (e) { - locations = {}; + entries = {}; } - var entry; - for ( location in locations ) { - if ( locations.hasOwnProperty(location) === false ) { - continue; - } - entry = locations[location]; - location = fixLocation(location); - // Migrate obsolete location to new location, if any - if ( typeof entry.oldLocation === 'string' ) { - entry.oldLocation = fixLocation(entry.oldLocation); - relocationMap[entry.oldLocation] = location; - } - availableLists[location] = entry; + for ( var assetKey in entries ) { + if ( entries.hasOwnProperty(assetKey) === false ) { continue; } + entry = entries[assetKey]; + if ( entry.content !== 'filters' ) { continue; } + newAvailableLists[assetKey] = objectAssign({}, entry); } - // Now get user's selection of lists - vAPI.storage.get( - { 'remoteBlacklists': availableLists }, - onSelectedListsLoaded - ); + // Load set of currently selected filter lists. + µb.loadSelectedFilterLists(onSelectedListsLoaded); }; - // permanent lists - var location; - var lists = this.permanentLists; - for ( location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; - } - availableLists[location] = lists[location]; - } - - // custom lists - var c; - var locations = this.userSettings.externalLists.split('\n'); - for ( var i = 0; i < locations.length; i++ ) { - location = locations[i].trim(); - c = location.charAt(0); - if ( location === '' || c === '!' || c === '#' ) { - continue; - } - // Coarse validation - if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) { - continue; - } - availableLists[location] = { - title: '', - group: 'custom', - external: true - }; - } - - // get built-in block lists. - this.loadAndPatchStockFilterLists(onBuiltinListsLoaded); -}; - -/******************************************************************************/ - -µBlock.autoSelectFilterLists = function(lists) { - var lang = self.navigator.language.slice(0, 2), - list; - for ( var path in lists ) { - if ( lists.hasOwnProperty(path) === false ) { - continue; - } - list = lists[path]; - if ( list.off !== true ) { - continue; - } - if ( list.lang === lang ) { - list.off = false; - } - } -}; - -/******************************************************************************/ + // Available lists previously computed. + var onOldAvailableListsLoaded = function(bin) { + oldAvailableLists = bin && bin.availableFilterLists || {}; + µb.assets.get('assets.json', onBuiltinListsLoaded); + }; -µBlock.createShortUniqueId = function(path) { - var md5 = YaMD5.hashStr(path); - return md5.slice(0, 4) + md5.slice(-4); + // Load previously saved available lists -- these contains data + // computed at run-time, we will reuse this data if possible. + vAPI.storage.get('availableFilterLists', onOldAvailableListsLoaded); }; -µBlock.createShortUniqueId.idLength = 8; - /******************************************************************************/ // This is used to be re-entrancy resistant. @@ -444,18 +430,11 @@ callback = this.noopFunc; } - // Never fetch from remote servers when we load filter lists: this has to - // be as fast as possible. - µb.assets.remoteFetchBarrier += 1; - var onDone = function() { - // Remove barrier to remote fetching - µb.assets.remoteFetchBarrier -= 1; - µb.staticNetFilteringEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); µb.redirectEngine.freeze(); - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists }); //quickProfiler.stop(0); @@ -473,15 +452,15 @@ var acceptedCount = snfe.acceptedCount + cfe.acceptedCount; var discardedCount = snfe.discardedCount + cfe.discardedCount; µb.applyCompiledFilters(compiled, path === µb.userFiltersPath); - if ( µb.remoteBlacklists.hasOwnProperty(path) ) { - var entry = µb.remoteBlacklists[path]; + if ( µb.availableFilterLists.hasOwnProperty(path) ) { + var entry = µb.availableFilterLists[path]; entry.entryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; entry.entryUsedCount = entry.entryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); } }; var onCompiledListLoaded = function(details) { - applyCompiledFilters(details.path, details.content); + applyCompiledFilters(details.assetKey, details.content); filterlistsCount -= 1; if ( filterlistsCount === 0 ) { onDone(); @@ -489,7 +468,7 @@ }; var onFilterListsReady = function(lists) { - µb.remoteBlacklists = lists; + µb.availableFilterLists = lists; µb.redirectEngine.reset(); µb.cosmeticFilteringEngine.reset(); @@ -502,14 +481,10 @@ // This happens for assets which do not exist, ot assets with no // content. var toLoad = []; - for ( var path in lists ) { - if ( lists.hasOwnProperty(path) === false ) { - continue; - } - if ( lists[path].off ) { - continue; - } - toLoad.push(path); + for ( var assetKey in lists ) { + if ( lists.hasOwnProperty(assetKey) === false ) { continue; } + if ( lists[assetKey].off ) { continue; } + toLoad.push(assetKey); } filterlistsCount = toLoad.length; if ( filterlistsCount === 0 ) { @@ -528,32 +503,17 @@ /******************************************************************************/ -µBlock.getCompiledFilterListPath = function(path) { - return 'cache://compiled-filter-list:' + this.createShortUniqueId(path); -}; - -/******************************************************************************/ - -µBlock.getCompiledFilterList = function(path, callback) { - var compiledPath = this.getCompiledFilterListPath(path); - var µb = this; +µBlock.getCompiledFilterList = function(assetKey, callback) { + var µb = this, + compiledPath = 'compiled/' + assetKey; var onRawListLoaded = function(details) { + details.assetKey = assetKey; if ( details.content === '' ) { callback(details); return; } - var listMeta = µb.remoteBlacklists[path]; - // https://github.com/gorhill/uBlock/issues/313 - // Always try to fetch the name if this is an external filter list. - if ( listMeta && (listMeta.title === '' || listMeta.group === 'custom') ) { - var matches = details.content.slice(0, 1024).match(/(?:^|\n)!\s*Title:([^\n]+)/i); - if ( matches !== null ) { - listMeta.title = matches[1].trim(); - } - } - - //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path); + µb.extractFilterListMetadata(assetKey, details.content); details.content = µb.compileFilters(details.content); µb.assets.put(compiledPath, details.content); callback(details); @@ -561,12 +521,10 @@ var onCompiledListLoaded = function(details) { if ( details.content === '' ) { - //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: no compiled version for "%s"', path); - µb.assets.get(path, onRawListLoaded); + µb.assets.get(assetKey, onRawListLoaded); return; } - //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: using compiled version for "%s"', path); - details.path = path; + details.assetKey = assetKey; callback(details); }; @@ -575,61 +533,71 @@ /******************************************************************************/ -µBlock.purgeCompiledFilterList = function(path) { - this.assets.purge(this.getCompiledFilterListPath(path)); +µBlock.extractFilterListMetadata = function(assetKey, raw) { + var listEntry = this.availableFilterLists[assetKey]; + if ( listEntry === undefined ) { return; } + // Metadata expected to be found at the top of content. + var head = raw.slice(0, 1024), + matches, v; + // https://github.com/gorhill/uBlock/issues/313 + // Always try to fetch the name if this is an external filter list. + if ( listEntry.title === '' || listEntry.group === 'custom' ) { + matches = head.match(/(?:^|\n)!\s*Title:([^\n]+)/i); + if ( matches !== null ) { + listEntry.title = matches[1].trim(); + } + } + // Extract update frequency information + matches = head.match(/(?:^|\n)![\t ]*?Expires:[\t ]*?([\d]+)/i); + if ( matches !== null ) { + v = Math.max(parseInt(matches[1], 10), 2); + if ( v !== listEntry.updateAfter ) { + listEntry.updateAfter = v; + this.assets.registerAssetSource(assetKey, listEntry); + } + } }; /******************************************************************************/ -µBlock.purgeFilterList = function(path) { - this.purgeCompiledFilterList(path); - this.assets.purge(path); +µBlock.removeCompiledFilterList = function(assetKey) { + this.assets.remove('compiled/' + assetKey); +}; + +µBlock.removeFilterList = function(assetKey) { + this.removeCompiledFilterList(assetKey); + this.assets.remove(assetKey); }; /******************************************************************************/ µBlock.compileFilters = function(rawText) { - var rawEnd = rawText.length; var compiledFilters = []; // Useful references: // https://adblockplus.org/en/filter-cheatsheet // https://adblockplus.org/en/filters - var staticNetFilteringEngine = this.staticNetFilteringEngine; - var cosmeticFilteringEngine = this.cosmeticFilteringEngine; - var reIsWhitespaceChar = /\s/; - var reMaybeLocalIp = /^[\d:f]/; - var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/; - var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/; - - var lineBeg = 0, lineEnd, currentLineBeg; - var line, lineRaw, c, pos; - - while ( lineBeg < rawEnd ) { - lineEnd = rawText.indexOf('\n', lineBeg); - if ( lineEnd === -1 ) { - lineEnd = rawText.indexOf('\r', lineBeg); - if ( lineEnd === -1 ) { - lineEnd = rawEnd; - } - } + var staticNetFilteringEngine = this.staticNetFilteringEngine, + cosmeticFilteringEngine = this.cosmeticFilteringEngine, + reIsWhitespaceChar = /\s/, + reMaybeLocalIp = /^[\d:f]/, + reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/, + reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/, + line, lineRaw, c, pos, + lineIter = new this.LineIterator(rawText); + + while ( lineIter.eot() === false ) { + line = lineRaw = lineIter.next().trim(); // rhill 2014-04-18: The trim is important here, as without it there // could be a lingering `\r` which would cause problems in the // following parsing code. - line = lineRaw = rawText.slice(lineBeg, lineEnd).trim(); - currentLineBeg = lineBeg; - lineBeg = lineEnd + 1; - if ( line.length === 0 ) { - continue; - } + if ( line.length === 0 ) { continue; } // Strip comments c = line.charAt(0); - if ( c === '!' || c === '[' ) { - continue; - } + if ( c === '!' || c === '[' ) { continue; } // Parse or skip cosmetic filters // All cosmetic filters are caught here @@ -640,9 +608,7 @@ // Whatever else is next can be assumed to not be a cosmetic filter // Most comments start in first column - if ( c === '#' ) { - continue; - } + if ( c === '#' ) { continue; } // Catch comments somewhere on the line // Remove: @@ -663,15 +629,11 @@ // Ignore hosts file redirect configuration // 127.0.0.1 localhost // 255.255.255.255 broadcasthost - if ( reIsLocalhostRedirect.test(line) ) { - continue; - } + if ( reIsLocalhostRedirect.test(line) ) { continue; } line = line.replace(reLocalIp, '').trim(); } - if ( line.length === 0 ) { - continue; - } + if ( line.length === 0 ) { continue; } staticNetFilteringEngine.compile(line, compiledFilters); } @@ -699,55 +661,6 @@ /******************************************************************************/ -// `switches` contains the filter lists for which the switch must be revisited. - -µBlock.selectFilterLists = function(switches) { - switches = switches || {}; - - // Only the lists referenced by the switches are touched. - var filterLists = this.remoteBlacklists; - var entry, state, location; - var i = switches.length; - while ( i-- ) { - entry = switches[i]; - state = entry.off === true; - location = entry.location; - if ( filterLists.hasOwnProperty(location) === false ) { - if ( state !== true ) { - filterLists[location] = { off: state }; - } - continue; - } - if ( filterLists[location].off === state ) { - continue; - } - filterLists[location].off = state; - } - - vAPI.storage.set({ 'remoteBlacklists': filterLists }); -}; - -/******************************************************************************/ - -// Plain reload of all filters. - -µBlock.reloadAllFilters = function() { - var µb = this; - - // We are just reloading the filter lists: we do not want assets to update. - // TODO: probably not needed anymore, since filter lists are now always - // loaded without update => see `µb.assets.remoteFetchBarrier`. - this.assets.autoUpdate = false; - - var onFiltersReady = function() { - µb.assets.autoUpdate = µb.userSettings.autoUpdate; - }; - - this.loadFilterLists(onFiltersReady); -}; - -/******************************************************************************/ - µBlock.loadRedirectResources = function(callback) { var µb = this; @@ -762,40 +675,46 @@ callback(); }; - this.assets.get('assets/ublock/resources.txt', onResourcesLoaded); + this.assets.get('ublock-resources', onResourcesLoaded); }; /******************************************************************************/ µBlock.loadPublicSuffixList = function(callback) { - var µb = this; - var path = µb.pslPath; - var compiledPath = 'cache://compiled-publicsuffixlist'; + var µb = this, + assetKey = µb.pslAssetKey, + compiledAssetKey = 'compiled/' + assetKey; if ( typeof callback !== 'function' ) { callback = this.noopFunc; } var onRawListLoaded = function(details) { if ( details.content !== '' ) { - //console.debug('µBlock.loadPublicSuffixList/onRawListLoaded: compiling "%s"', path); - publicSuffixList.parse(details.content, punycode.toASCII); - µb.assets.put(compiledPath, JSON.stringify(publicSuffixList.toSelfie())); + µb.compilePublicSuffixList(details.content); } callback(); }; var onCompiledListLoaded = function(details) { if ( details.content === '' ) { - //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: no compiled version for "%s"', path); - µb.assets.get(path, onRawListLoaded); + µb.assets.get(assetKey, onRawListLoaded); return; } - //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: using compiled version for "%s"', path); publicSuffixList.fromSelfie(JSON.parse(details.content)); callback(); }; - this.assets.get(compiledPath, onCompiledListLoaded); + this.assets.get(compiledAssetKey, onCompiledListLoaded); +}; + +/******************************************************************************/ + +µBlock.compilePublicSuffixList = function(content) { + publicSuffixList.parse(content, punycode.toASCII); + this.assets.put( + 'compiled/' + this.pslAssetKey, + JSON.stringify(publicSuffixList.toSelfie()) + ); }; /******************************************************************************/ @@ -814,7 +733,7 @@ var selfie = { magic: µb.systemSettings.selfieMagic, publicSuffixList: publicSuffixList.toSelfie(), - filterLists: µb.remoteBlacklists, + availableFilterLists: µb.availableFilterLists, staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(), redirectEngine: µb.redirectEngine.toSelfie(), cosmeticFilteringEngine: µb.cosmeticFilteringEngine.toSelfie() @@ -899,7 +818,7 @@ } if ( typeof data.filterLists === 'object' ) { - bin.remoteBlacklists = data.filterLists; + bin.filterLists = data.filterLists; binNotEmpty = true; } @@ -939,203 +858,86 @@ /******************************************************************************/ -µBlock.updateStartHandler = function(callback) { - var µb = this; - var onListsReady = function(lists) { - var assets = {}; - for ( var location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; - } - if ( lists[location].off ) { - continue; - } - assets[location] = true; - } - assets[µb.pslPath] = true; - assets['assets/ublock/resources.txt'] = true; - callback(assets); - }; - - this.getAvailableLists(onListsReady); -}; - -/******************************************************************************/ - -µBlock.assetUpdatedHandler = function(details) { - var path = details.path || ''; - if ( this.remoteBlacklists.hasOwnProperty(path) === false ) { - return; - } - var entry = this.remoteBlacklists[path]; - if ( entry.off ) { - return; - } - // Compile the list while we have the raw version in memory - //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path); - this.assets.put( - this.getCompiledFilterListPath(path), - this.compileFilters(details.content) - ); -}; - -/******************************************************************************/ - -µBlock.updateCompleteHandler = function(details) { - var µb = this; - var updatedCount = details.updatedCount; - - // Assets are supposed to have been all updated, prevent fetching from - // remote servers. - µb.assets.remoteFetchBarrier += 1; - - var onFiltersReady = function() { - µb.assets.remoteFetchBarrier -= 1; - }; - - var onPSLReady = function() { - if ( updatedCount !== 0 ) { - //console.debug('storage.js > µBlock.updateCompleteHandler: reloading filter lists'); - µb.loadFilterLists(onFiltersReady); - } else { - onFiltersReady(); - } - }; - - if ( details.hasOwnProperty(this.pslPath) ) { - //console.debug('storage.js > µBlock.updateCompleteHandler: reloading PSL'); - this.loadPublicSuffixList(onPSLReady); - updatedCount -= 1; - } else { - onPSLReady(); - } -}; - -/******************************************************************************/ - -µBlock.assetCacheRemovedHandler = (function() { - var barrier = false; - - var handler = function(paths) { - if ( barrier ) { - return; - } - barrier = true; - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( this.remoteBlacklists.hasOwnProperty(path) ) { - //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path); - this.purgeCompiledFilterList(path); - continue; - } - if ( path === this.pslPath ) { - //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path); - this.assets.purge('cache://compiled-publicsuffixlist'); - continue; - } - } - this.selfieManager.destroy(); - barrier = false; +µBlock.scheduleAssetUpdater = (function() { + var timer, next = 0; + return function(updateDelay) { + if ( timer ) { + clearTimeout(timer); + timer = undefined; + next = 0; + } + if ( updateDelay === 0 ) { return; } + var now = Date.now(); + // Use the new schedule if and only if it is earlier than the previous + // one. + if ( next !== 0 ) { + updateDelay = Math.min(updateDelay, next - now); + } + next = now + updateDelay; + timer = vAPI.setTimeout(function() { + timer = undefined; + next = 0; + µBlock.assets.updateStart({ delay: 2 * 60 * 1000 }); + }, updateDelay); }; - - return handler; })(); /******************************************************************************/ -// https://github.com/gorhill/uBlock/issues/602 -// - Load and patch `filter-list.json` -// - Load and patch user's `remoteBlacklists` -// - Load and patch cached filter lists -// - Load and patch compiled filter lists -// -// Once enough time has passed to safely assume all uBlock Origin -// installations have been converted to the new stock filter lists, this code -// can be removed. - -µBlock.patchFilterLists = function(filterLists) { - var modified = false; - var oldListKey, newListKey, listEntry; - for ( var listKey in filterLists ) { - if ( filterLists.hasOwnProperty(listKey) === false ) { - continue; - } - oldListKey = listKey; - if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - oldListKey = 'assets/thirdparties/' + listKey; - if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - continue; - } - } - newListKey = this.oldListToNewListMap[oldListKey]; - // https://github.com/gorhill/uBlock/issues/668 - // https://github.com/gorhill/uBlock/issues/669 - // Beware: an entry for the new list key may already exists. If it is - // the case, leave it as is. - if ( newListKey !== '' && filterLists.hasOwnProperty(newListKey) === false ) { - listEntry = filterLists[listKey]; - listEntry.homeURL = undefined; - filterLists[newListKey] = listEntry; +µBlock.assetObserver = function(topic, details) { + // Do not update filter list if not in use. + if ( topic === 'before-asset-updated' ) { + if ( + this.availableFilterLists.hasOwnProperty(details.assetKey) && + this.availableFilterLists[details.assetKey].off === true + ) { + return false; } - delete filterLists[listKey]; - modified = true; + return; } - return modified; -}; -µBlock.loadAndPatchStockFilterLists = function(callback) { - var onStockListsLoaded = function(details) { - var µb = µBlock; - var stockLists; - try { - stockLists = JSON.parse(details.content); - } catch (e) { - stockLists = {}; - } - - // Migrate assets affected by the change to their new name. - var reExternalURL = /^https?:\/\//; - var newListKey; - for ( var oldListKey in stockLists ) { - if ( stockLists.hasOwnProperty(oldListKey) === false ) { - continue; - } - // https://github.com/gorhill/uBlock/issues/708 - // Support migrating external stock filter lists as well. - if ( reExternalURL.test(oldListKey) === false ) { - oldListKey = 'assets/thirdparties/' + oldListKey; + // Compile the list while we have the raw version in memory + if ( topic === 'after-asset-updated' ) { + var cached = typeof details.content === 'string' && details.content !== ''; + if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) { + if ( cached ) { + if ( this.availableFilterLists[details.assetKey].off !== true ) { + this.extractFilterListMetadata( + details.assetKey, + details.content + ); + this.assets.put( + 'compiled/' + details.assetKey, + this.compileFilters(details.content) + ); + } + } else { + this.removeCompiledFilterList(details.assetKey); } - if ( µb.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - continue; + } else if ( details.assetKey === this.pslAssetKey ) { + if ( cached ) { + this.compilePublicSuffixList(details.content); } - newListKey = µb.oldListToNewListMap[oldListKey]; - if ( newListKey === '' ) { - continue; + } else if ( details.assetKey === 'ublock-resources' ) { + if ( cached ) { + this.redirectEngine.resourcesFromString(details.content); } - // Rename cached asset to preserve content -- so it does not - // need to be fetched from remote server. - µb.assets.rename(oldListKey, newListKey); - µb.assets.purge(µb.getCompiledFilterListPath(oldListKey)); } - µb.patchFilterLists(stockLists); - - // Stock lists information cascades into - // - In-memory user's selected filter lists, so we need to patch this. - µb.patchFilterLists(µb.remoteBlacklists); - - // Stock lists information cascades into - // - In-storage user's selected filter lists, so we need to patch this. - vAPI.storage.get('remoteBlacklists', function(bin) { - var userLists = bin.remoteBlacklists || {}; - if ( µb.patchFilterLists(userLists) ) { - µb.keyvalSetOne('remoteBlacklists', userLists); - } - details.content = JSON.stringify(stockLists); - callback(details); + vAPI.messaging.broadcast({ + what: 'assetUpdated', + key: details.assetKey, + cached: cached + }); - }; + return; + } - this.assets.get('assets/ublock/filter-lists.json', onStockListsLoaded); + // Reload all filter lists if needed. + if ( topic === 'after-assets-updated' ) { + if ( details.assetKeys.length !== 0 ) { + this.loadFilterLists(); + } + this.scheduleAssetUpdater(this.userSettings.autoUpdate ? 11 * 60 * 60 * 1000 : -1); + return; + } }; diff --git a/src/js/traffic.js b/src/js/traffic.js index 1c67e0de52c28..b74cdfbf74697 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -495,7 +495,7 @@ var processCSP = function(details, pageStore, context) { ); } - if ( loggerEnabled && blockWebsocket ) { + if ( loggerEnabled && websocketResult !== '' ) { µb.logger.writeOne( tabId, 'net', diff --git a/src/js/ublock.js b/src/js/ublock.js index 528edb534897f..f765fbd74d4b8 100644 --- a/src/js/ublock.js +++ b/src/js/ublock.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -317,6 +317,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/, // Pre-change switch ( name ) { + case 'externalLists': + this.changeExternalFilterLists(us.externalLists, value); + break; case 'largeMediaSize': if ( typeof value !== 'number' ) { value = parseInt(value, 10) || 0; @@ -340,6 +343,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/, us.dynamicFilteringEnabled = true; } break; + case 'autoUpdate': + this.scheduleAssetUpdater(value ? 7 * 60 * 1000 : 0); + break; case 'collapseBlocked': if ( value === false ) { this.cosmeticFilteringEngine.removeFromSelectorCache('*', 'net'); diff --git a/tools/make-assets.sh b/tools/make-assets.sh index fade452dcbafe..cebea3f454e8e 100755 --- a/tools/make-assets.sh +++ b/tools/make-assets.sh @@ -24,8 +24,6 @@ cp -R ../uAssets/thirdparties/www.malwaredomainlist.com $DES/thirdparti mkdir $DES/ublock cp -R ../uAssets/filters/* $DES/ublock/ -cp -R ./assets/ublock/filter-lists.json $DES/ublock/ - -cp ../uAssets/checksums/ublock0.txt $DES/checksums.txt +cp -R ./assets/assets.json $DES/ echo "done." diff --git a/tools/make-chromium.sh b/tools/make-chromium.sh index 8d1f58911131a..eda30f2991edf 100755 --- a/tools/make-chromium.sh +++ b/tools/make-chromium.sh @@ -5,7 +5,11 @@ echo "*** uBlock0.chromium: Creating web store package" echo "*** uBlock0.chromium: Copying files" -DES=dist/build/uBlock0.chromium +if [ "$1" = experimental ]; then + DES=dist/build/experimental/uBlock0.chromium +else + DES=dist/build/uBlock0.chromium +fi rm -rf $DES mkdir -p $DES From bf29bae78c3d261ad555a8ead41cd8dc090d9982 Mon Sep 17 00:00:00 2001 From: gorhill Date: Sun, 15 Jan 2017 18:58:38 -0500 Subject: [PATCH 02/15] finalizing refactoring of assets management --- assets/assets.json | 10 +- src/js/3p-filters.js | 131 ++++++++++---------- src/js/assets.js | 278 +++++++++++++++++++++++++++++++------------ src/js/messaging.js | 41 ++++--- src/js/settings.js | 5 +- src/js/storage.js | 97 ++++++++++----- 6 files changed, 358 insertions(+), 204 deletions(-) diff --git a/assets/assets.json b/assets/assets.json index 8a165b43ef90e..6ba6af58933e0 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -253,8 +253,8 @@ "off": true, "title": "hpHosts’ Ad and tracking servers", "group": "multipurpose", - "contentURL": "http://hosts-file.net/.%5Cad_servers.txt", - "supportURL": "http://hosts-file.net/" + "contentURL": "https://hosts-file.net/.%5Cad_servers.txt", + "supportURL": "https://hosts-file.net/" }, "EST-0": { "content": "filters", @@ -405,7 +405,8 @@ "group": "malware", "contentURL": [ "https://mirror.cedia.org.ec/malwaredomains/justdomains", - "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains" + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt" ], "supportURL": "http://www.malwaredomains.com/" }, @@ -415,7 +416,8 @@ "group": "multipurpose", "contentURL": [ "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext", - "assets/thirdparties/pgl.yoyo.org/as/serverlist" + "assets/thirdparties/pgl.yoyo.org/as/serverlist", + "assets/thirdparties/pgl.yoyo.org/as/serverlist.txt" ], "supportURL": "https://pgl.yoyo.org/adservers/" }, diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index fc2d2509b4418..e4339b8da91ce 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -32,6 +32,7 @@ var listDetails = {}; var parseCosmeticFilters = true; var ignoreGenericCosmeticFilters = false; +var selectedListsHashBefore = ''; var externalLists = ''; /******************************************************************************/ @@ -61,13 +62,13 @@ var renderNumber = function(value) { /******************************************************************************/ var renderFilterLists = function() { - var listGroupTemplate = uDom('#templates .groupEntry'); - var listEntryTemplate = uDom('#templates .listEntry'); - var listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'); - var renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString; - var lastUpdateString = vAPI.i18n('3pLastUpdate'); + var listGroupTemplate = uDom('#templates .groupEntry'), + listEntryTemplate = uDom('#templates .listEntry'), + listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'), + renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString, + lastUpdateString = vAPI.i18n('3pLastUpdate'); - // Assemble a pretty blacklist name if possible + // Assemble a pretty list name if possible var listNameFromListKey = function(listKey) { var list = listDetails.current[listKey] || listDetails.available[listKey]; var listTitle = list ? list.title : ''; @@ -81,6 +82,7 @@ var renderFilterLists = function() { var entry = listDetails.available[listKey]; var li = listEntryTemplate.clone(); + li.attr('data-listkey', listKey); if ( entry.off !== true ) { li.descendants('input').attr('checked', ''); } @@ -88,7 +90,6 @@ var renderFilterLists = function() { var elem = li.descendants('a:nth-of-type(1)'); elem.attr('href', 'asset-viewer.html?url=' + encodeURI(listKey)); elem.attr('type', 'text/html'); - elem.attr('data-listkey', listKey); elem.text(listNameFromListKey(listKey) + '\u200E'); if ( entry.instructionURL ) { @@ -236,6 +237,15 @@ var renderFilterLists = function() { .replace('{{cosmeticFilterCount}}', renderNumber(details.cosmeticFilterCount)) ); + // Compute a hash of the lists currently enabled in memory. + var selectedListsBefore = []; + for ( var key in listDetails.current ) { + if ( listDetails.current[key].off !== true ) { + selectedListsBefore.push(key); + } + } + selectedListsHashBefore = selectedListsBefore.sort().join(); + renderWidgets(); }; @@ -254,7 +264,7 @@ var renderWidgets = function() { /******************************************************************************/ var updateAssetStatus = function(details) { - var li = uDom('#lists .listEntry a[data-listkey="' + details.key + '"]').ancestors('.listEntry'); + var li = uDom('#lists .listEntry[data-listkey="' + details.key + '"]'); li.toggleClass('obsolete', !details.cached); li.toggleClass('cached', details.cached); }; @@ -271,46 +281,27 @@ var listsSelectionChanged = function() { ) { return true; } - - var availableLists = listDetails.available, - currentLists = listDetails.current, - listKey, availableOff, currentOff; - - // This checks existing entries - for ( listKey in availableLists ) { - if ( availableLists.hasOwnProperty(listKey) === false ) { continue; } - availableOff = availableLists[listKey].off === true; - currentOff = currentLists[listKey] === undefined || currentLists[listKey].off === true; - if ( availableOff !== currentOff ) { return true; } - } - - // This checks removed entries - for ( listKey in currentLists ) { - if ( currentLists.hasOwnProperty(listKey) === false ) { continue; } - currentOff = currentLists[listKey].off === true; - availableOff = availableLists[listKey] === undefined || availableLists[listKey].off === true; - if ( availableOff !== currentOff ) { return true; } + var selectedListsAfter = [], + listEntries = uDom('#lists .listEntry[data-listkey] > input[type="checkbox"]:checked'); + for ( var i = 0, n = listEntries.length; i < n; i++ ) { + selectedListsAfter.push(listEntries.at(i).ancestors('.listEntry[data-listkey]').attr('data-listkey')); } - return false; + return selectedListsHashBefore !== selectedListsAfter.sort().join(); }; /******************************************************************************/ var onListCheckboxChanged = function() { - var href = uDom(this).parent().descendants('a').first().attr('data-listkey'); - if ( typeof href !== 'string' ) { return; } - if ( listDetails.available[href] === undefined ) { return; } - listDetails.available[href].off = !this.checked; renderWidgets(); }; /******************************************************************************/ var onPurgeClicked = function() { - var button = uDom(this); - var li = button.parent(); - var listKey = li.descendants('a').first().attr('data-listkey'); + var button = uDom(this), + liEntry = button.ancestors('[data-listkey]'), + listKey = liEntry.attr('data-listkey'); if ( !listKey ) { return; } messaging.send('dashboard', { what: 'purgeCache', path: listKey }); @@ -323,12 +314,12 @@ var onPurgeClicked = function() { // be fetched anyways if there is no cached copy. var entry = listDetails.current && listDetails.current[listKey]; if ( entry && entry.off !== true ) { - li.addClass('obsolete'); + liEntry.addClass('obsolete'); uDom('#buttonUpdate').removeClass('disabled'); } - li.removeClass('cached'); + liEntry.removeClass('cached'); - if ( li.descendants('input').first().prop('checked') ) { + if ( liEntry.descendants('input').first().prop('checked') ) { renderWidgets(); } }; @@ -350,12 +341,12 @@ var selectFilterLists = function(callback) { // Filter lists var listKeys = [], - lis = uDom('#lists .listEntry'), li, - i = lis.length; + liEntries = uDom('#lists .listEntry'), liEntry, + i = liEntries.length; while ( i-- ) { - li = lis.at(i); - if ( li.descendants('input').prop('checked') ) { - listKeys.push(li.descendants('a').attr('data-listkey')); + liEntry = liEntries.at(i); + if ( liEntry.descendants('input').first().prop('checked') ) { + listKeys.push(liEntry.attr('data-listkey')); } } @@ -480,7 +471,7 @@ var groupEntryClickHandler = function() { /******************************************************************************/ -var getCloudData = function() { +var toCloudData = function() { var bin = { parseCosmeticFilters: uDom.nodeFromId('parseCosmeticFilters').checked, ignoreGenericCosmeticFilters: uDom.nodeFromId('ignoreGenericCosmeticFilters').checked, @@ -488,24 +479,22 @@ var getCloudData = function() { externalLists: externalLists }; - var lis = uDom('#lists .listEntry'), li; - var i = lis.length; + var liEntries = uDom('#lists .listEntry'), liEntry; + var i = liEntries.length; while ( i-- ) { - li = lis.at(i); - if ( li.descendants('input').prop('checked') ) { - bin.selectedLists.push(li.descendants('a').attr('data-listkey')); + liEntry = liEntries.at(i); + if ( liEntry.descendants('input').prop('checked') ) { + bin.selectedLists.push(liEntry.attr('data-listkey')); } } return bin; }; -var setCloudData = function(data, append) { - if ( typeof data !== 'object' || data === null ) { - return; - } +var fromCloudData = function(data, append) { + if ( typeof data !== 'object' || data === null ) { return; } - var elem, checked; + var elem, checked, i, n; elem = uDom.nodeFromId('parseCosmeticFilters'); checked = data.parseCosmeticFilters === true || append && elem.checked; @@ -515,30 +504,34 @@ var setCloudData = function(data, append) { checked = data.ignoreGenericCosmeticFilters === true || append && elem.checked; elem.checked = listDetails.ignoreGenericCosmeticFilters = checked; - var lis = uDom('#lists .listEntry'), li, listKey; - var i = lis.length; - while ( i-- ) { - li = lis.at(i); - elem = li.descendants('input'); - listKey = li.descendants('a').attr('data-listkey'); - checked = data.selectedLists.indexOf(listKey) !== -1 || - append && elem.prop('checked'); - elem.prop('checked', checked); - listDetails.available[listKey].off = !checked; + var listKey; + for ( i = 0, n = data.selectedLists.length; i < n; i++ ) { + listKey = data.selectedLists[i]; + if ( listDetails.aliases[listKey] ) { + data.selectedLists[i] = listDetails.aliases[listKey]; + } + } + var selectedSet = new Set(data.selectedLists), + listEntries = uDom('#lists .listEntry'), + listEntry, input; + for ( i = 0, n = listEntries.length; i < n; i++ ) { + listEntry = listEntries.at(i); + listKey = listEntry.attr('data-listkey'); + input = listEntry.descendants('input').first(); + if ( append && input.prop('checked') ) { continue; } + input.prop('checked', selectedSet.has(listKey) ); } elem = uDom.nodeFromId('externalLists'); - if ( !append ) { - elem.value = ''; - } + if ( !append ) { elem.value = ''; } elem.value += data.externalLists || ''; renderWidgets(); externalListsChangeHandler(); }; -self.cloud.onPush = getCloudData; -self.cloud.onPull = setCloudData; +self.cloud.onPush = toCloudData; +self.cloud.onPull = fromCloudData; /******************************************************************************/ diff --git a/src/js/assets.js b/src/js/assets.js index 91c91b2072721..34e661e2048dd 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -116,6 +116,149 @@ var getTextFileFromURL = function(url, onLoad, onError) { } }; +/******************************************************************************* + + TODO(seamless migration): + This block of code will be removed when I am confident all users have + moved to a version of uBO which does not require the old way of caching + assets. + + api.listKeyAliases: a map of old asset keys to new asset keys. + + migrate(): to seamlessly migrate the old cache manager to the new one: + - attempt to preserve and move content of cached assets to new locations; + - removes all traces of now obsolete cache manager entries in cacheStorage. + + This code will typically execute only once, when the newer version of uBO + is first installed and executed. + +**/ + +api.listKeyAliases = { + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat", + "assets/user/filters.txt": "user-filters", + "assets/ublock/resources.txt": "ublock-resources", + "assets/ublock/filters.txt": "ublock-filters", + "assets/ublock/privacy.txt": "ublock-privacy", + "assets/ublock/unbreak.txt": "ublock-unbreak", + "assets/ublock/badware.txt": "ublock-badware", + "assets/ublock/experimental.txt": "ublock-experimental", + "https://easylist-downloads.adblockplus.org/easylistchina.txt": "CHN-0", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "CHN-1", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt": "CHN-2", + "https://easylist-downloads.adblockplus.org/easylistgermany.txt": "DEU-0", + "https://adblock.dk/block.csv": "DNK-0", + "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt": "easylist", + "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "easylist-nocosmetic", + "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt": "easyprivacy", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "fanboy-annoyance", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt": "fanboy-social", + "https://easylist-downloads.adblockplus.org/liste_fr.txt": "FRA-0", + "http://adblock.gardar.net/is.abp.txt": "ISL-0", + "https://easylist-downloads.adblockplus.org/easylistitaly.txt": "ITA-0", + "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "ITA-1", + "https://easylist-downloads.adblockplus.org/advblock.txt": "RUS-0", + "https://easylist-downloads.adblockplus.org/bitblock.txt": "RUS-1", + "https://filters.adtidy.org/extension/chromium/filters/1.txt": "RUS-2", + "https://adguard.com/en/filter-rules.html?id=1": "RUS-2", + "https://easylist-downloads.adblockplus.org/easylistdutch.txt": "NLD-0", + "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt": "LVA-0", + "http://hosts-file.net/.%5Cad_servers.txt": "hphosts", + "http://adblock.ee/list.php": "EST-0", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt": "disconnect-malvertising", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt": "disconnect-malware", + "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt": "disconnect-tracking", + "https://www.certyficate.it/adblock/adblock.txt": "POL-0", + "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt": "awrl-0", + "http://adb.juvander.net/Finland_adb.txt": "FIN-0", + "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt": "KOR-0", + "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt": "KOR-1", + "https://www.fanboy.co.nz/fanboy-korean.txt": "KOR-2", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "IDN-0", + "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt": "JPN-0", + "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt": "EU-prebake", + "https://easylist-downloads.adblockplus.org/Liste_AR.txt": "ara-0", + "http://margevicius.lt/easylistlithuania.txt": "LTU-0", + "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-0", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-1", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-2", + "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", + "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt": "ISR-0", + "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "reek-0", + "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "HUN-0", + "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "CZE-0", + "http://someonewhocares.org/hosts/hosts": "dpollock-0", + "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt": "spam404-0", + "http://stanev.org/abp/adblock_bg.txt": "BGR-0", + "http://winhelp2002.mvps.org/hosts.txt": "mvps-0", + "https://www.fanboy.co.nz/enhancedstats.txt": "fanboy-enhanced", + "https://www.fanboy.co.nz/fanboy-antifacebook.txt": "fanboy-thirdparty_social", + "https://easylist-downloads.adblockplus.org/easylistspanish.txt": "spa-0", + "https://www.fanboy.co.nz/fanboy-swedish.txt": "SWE-0", + "https://www.fanboy.co.nz/r/fanboy-ultimate.txt": "fanboy-ultimate", + "https://filters.adtidy.org/extension/chromium/filters/13.txt": "TUR-0", + "https://adguard.com/filter-rules.html?id=13": "TUR-0", + "https://www.fanboy.co.nz/fanboy-vietnam.txt": "VIE-0", + "https://www.void.gr/kargig/void-gr-filters.txt": "GRC-0", + "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt": "SVN-0" +}; + +var migrate = function(callback) { + var entries, + moveCount = 0, + toRemove = []; + + var countdown = function(change) { + moveCount -= (change || 0); + if ( moveCount !== 0 ) { return; } + vAPI.cacheStorage.remove(toRemove); + saveAssetCacheRegistry(); + callback(); + }; + + var onContentRead = function(oldKey, newKey, bin) { + var content = bin && bin['cached_asset_content://' + oldKey] || undefined; + if ( content ) { + assetCacheRegistry[newKey] = { + readTime: Date.now(), + writeTime: entries[oldKey] + }; + if ( reIsExternalPath.test(oldKey) ) { + assetCacheRegistry[newKey].remoteURL = oldKey; + } + bin = {}; + bin['cache/' + newKey] = content; + vAPI.cacheStorage.set(bin); + } + countdown(1); + }; + + var onEntries = function(bin) { + entries = bin && bin['cached_asset_entries']; + if ( !entries ) { return callback(); } + var aliases = api.listKeyAliases; + for ( var oldKey in entries ) { + if ( oldKey.endsWith('assets/user/filters.txt') ) { continue; } + var newKey = aliases[oldKey]; + if ( !newKey && /^https?:\/\//.test(oldKey) ) { + newKey = oldKey; + } + if ( newKey ) { + vAPI.cacheStorage.get( + 'cached_asset_content://' + oldKey, + onContentRead.bind(null, oldKey, newKey) + ); + moveCount += 1; + } + toRemove.push('cached_asset_content://' + oldKey); + } + toRemove.push('cached_asset_entries', 'extensionLastVersion'); + countdown(); + }; + + vAPI.cacheStorage.get('cached_asset_entries', onEntries); +}; + /******************************************************************************* The purpose of the asset source registry is to keep key detail information @@ -132,40 +275,45 @@ var getTextFileFromURL = function(url, onLoad, onError) { var assetSourceRegistryStatus, assetSourceRegistry = Object.create(null); -var registerAssetSource = function(assetKey, details) { - var contentURL = details.contentURL || []; - if ( typeof contentURL === 'string' ) { - contentURL = [ contentURL ]; - } - var updateAfter = details.updateAfter || 11; - var submitter = details.submitter || undefined; - var entry = assetSourceRegistry[assetKey]; - if ( entry === undefined ) { - entry = Object.create(null); +var registerAssetSource = function(assetKey, dict) { + var entry = assetSourceRegistry[assetKey] || {}; + for ( var prop in dict ) { + if ( dict.hasOwnProperty(prop) === false ) { continue; } + if ( dict[prop] === undefined ) { + delete entry[prop]; + } else { + entry[prop] = dict[prop]; + } } - var remoteURLCount = 0; - for ( var i = 0; i < contentURL.length; i++ ) { - if ( reIsExternalPath.test(contentURL[i]) ) { - remoteURLCount += 1; + var contentURL = dict.contentURL; + if ( contentURL !== undefined ) { + if ( typeof contentURL === 'string' ) { + contentURL = entry.contentURL = [ contentURL ]; + } else if ( Array.isArray(contentURL) === false ) { + contentURL = entry.contentURL = []; + } + var remoteURLCount = 0; + for ( var i = 0; i < contentURL.length; i++ ) { + if ( reIsExternalPath.test(contentURL[i]) ) { + remoteURLCount += 1; + } } + entry.hasLocalURL = remoteURLCount !== contentURL.length; + entry.hasRemoteURL = remoteURLCount !== 0; + } else if ( entry.contentURL === undefined ) { + entry.contentURL = []; + } + if ( typeof entry.updateAfter !== 'number' ) { + entry.updateAfter = 13; } - entry.contentURL = contentURL; - entry.hasLocalURL = remoteURLCount !== contentURL.length; - entry.hasRemoteURL = remoteURLCount !== 0; - entry.updateAfter = updateAfter; - entry.submitter = submitter; if ( entry.submitter ) { entry.submitTime = Date.now(); // To detect stale entries } assetSourceRegistry[assetKey] = entry; }; -var registerAssetError = function(assetKey, details) { - getAssetSourceRegistry(function() { - var entry = assetSourceRegistry[assetKey]; - if ( entry === undefined ) { return; } - entry.error = details; - }); +var unregisterAssetSource = function(assetKey) { + delete assetSourceRegistry[assetKey]; }; var saveAssetSourceRegistry = (function() { @@ -239,16 +387,17 @@ var getAssetSourceRegistry = function(callback) { //var updateAssetSourceRegistry = function(raw) {}; -api.getAssetSourceRegistry = function(callback) { +api.registerAssetSource = function(assetKey, details) { getAssetSourceRegistry(function() { - callback(JSON.parse(JSON.stringify(assetSourceRegistry))); + registerAssetSource(assetKey, details); + saveAssetSourceRegistry(true); }); }; -api.registerAssetSource = function(assetKey, details) { +api.unregisterAssetSource = function(assetKey) { getAssetSourceRegistry(function() { - registerAssetSource(assetKey, details); - saveAssetSourceRegistry(); + unregisterAssetSource(assetKey); + saveAssetSourceRegistry(true); }); }; @@ -290,8 +439,10 @@ var getAssetCacheRegistry = function(callback) { vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { if ( bin && bin.assetCacheRegistry ) { assetCacheRegistry = bin.assetCacheRegistry; + registryReady(); + } else { + migrate(registryReady); } - registryReady(); }); }; @@ -302,9 +453,7 @@ var saveAssetCacheRegistry = (function() { vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry }); }; return function(lazily) { - if ( timer !== undefined ) { - clearTimeout(timer); - } + if ( timer !== undefined ) { clearTimeout(timer); } if ( lazily ) { timer = vAPI.setTimeout(save, 500); } else { @@ -447,53 +596,24 @@ var stringIsNotEmpty = function(s) { return typeof s === 'string' && s !== ''; }; -/******************************************************************************* - - TODO: This will be removed when I am confident all users have moved - to a version of uBO which does not require the old way to persist cached - assets. - -**/ - -(function() { - vAPI.cacheStorage.get('cached_asset_entries', function(bin) { - var entries = bin && bin['cached_asset_entries']; - if ( !entries ) { return; } - var keystoRemove = [], - paths = Object.keys(entries), - i = paths.length, - path; - while ( i-- ) { - path = paths[i]; - if ( path === 'assets/user/filters.txt' ) { continue; } - keystoRemove.push('cached_asset_content://' + path); - } - keystoRemove.push('extensionLastVersion'); - keystoRemove.push('cached_asset_entries'); - vAPI.cacheStorage.remove(keystoRemove); - }); -})(); - /******************************************************************************* User assets are NOT persisted in the cache storage. User assets are recognized by the asset key which always starts with 'user-'. - TODO: Revisit when I am confident all users have moved to a version of uBO - which does not require the old way to persist user assets. + TODO(seamless migration): + Can remove instances of old user asset keys when I am confident all users + are using uBO v1.11 and beyond. **/ var readUserAsset = function(assetKey, callback) { - // TODO: remove old cruft when confident all users moved to uBO 1.11.0+. var reportBack = function(content) { callback({ assetKey: assetKey, content: content }); }; var onLoaded = function(bin) { - if ( !bin ) { - return reportBack(''); - } + if ( !bin ) { return reportBack(''); } var content = ''; if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) { content = bin['cached_asset_content://assets/user/filters.txt']; @@ -505,18 +625,22 @@ var readUserAsset = function(assetKey, callback) { } if ( typeof bin[assetKey] === 'string' ) { content = bin[assetKey]; + } else if ( content !== '' ) { + bin = {}; + bin[assetKey] = content; + vAPI.storage.set(bin); } return reportBack(content); }; - - vAPI.storage.get( - [ + var toRead = assetKey; + if ( assetKey === µBlock.userFiltersPath ) { + toRead = [ assetKey, 'assets/user/filters.txt', 'cached_asset_content://assets/user/filters.txt' - ], - onLoaded - ); + ]; + } + vAPI.storage.get(toRead, onLoaded); }; var saveUserAsset = function(assetKey, content, callback) { @@ -533,7 +657,7 @@ var saveUserAsset = function(assetKey, content, callback) { /******************************************************************************/ api.get = function(assetKey, callback) { - if ( reIsUserAsset.test(assetKey) ) { + if ( assetKey === µBlock.userFiltersPath ) { readUserAsset(assetKey, callback); return; } @@ -623,7 +747,7 @@ var getRemote = function(assetKey, callback) { var onRemoteContentLoaded = function() { if ( stringIsNotEmpty(this.responseText) === false ) { - registerAssetError(assetKey, { time: Date.now(), error: 'No content' }); + registerAssetSource(assetKey, { error: { time: Date.now(), error: 'No content' } }); tryLoading(); return; } @@ -631,12 +755,12 @@ var getRemote = function(assetKey, callback) { content: this.responseText, url: contentURL }); - registerAssetError(assetKey); + registerAssetSource(assetKey, { error: undefined }); reportBack(this.responseText); }; var onRemoteContentError = function() { - registerAssetError(assetKey, { time: Date.now(), error: this.statusText }); + registerAssetSource(assetKey, { error: { time: Date.now(), error: this.statusText } }); tryLoading(); }; diff --git a/src/js/messaging.js b/src/js/messaging.js index f71444b2f14b8..6b0970fb9a286 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -792,32 +792,32 @@ var backupUserData = function(callback) { var restoreUserData = function(request) { var userData = request.userData; - var countdown = 8; - var onCountdown = function() { - countdown -= 1; - if ( countdown === 0 ) { - vAPI.app.restart(); - } - }; var onAllRemoved = function() { - // Be sure to adjust `countdown` if adding/removing anything below - µb.keyvalSetOne('version', userData.version); µBlock.saveLocalSettings(); - vAPI.storage.set(userData.userSettings, onCountdown); - µb.keyvalSetOne('filterLists', userData.filterLists, onCountdown); + vAPI.storage.set(userData.userSettings); µb.hiddenSettingsFromString(userData.hiddenSettingsString || ''); - µb.keyvalSetOne('netWhitelist', userData.netWhitelist || '', onCountdown); - µb.keyvalSetOne('dynamicFilteringString', userData.dynamicFilteringString || '', onCountdown); - µb.keyvalSetOne('urlFilteringString', userData.urlFilteringString || '', onCountdown); - µb.keyvalSetOne('hostnameSwitchesString', userData.hostnameSwitchesString || '', onCountdown); - µb.assets.put(µb.userFiltersPath, userData.userFilters, onCountdown); vAPI.storage.set({ + netWhitelist: userData.netWhitelist || '', + dynamicFilteringString: userData.dynamicFilteringString || '', + urlFilteringString: userData.urlFilteringString || '', + hostnameSwitchesString: userData.hostnameSwitchesString || '', lastRestoreFile: request.file || '', lastRestoreTime: Date.now(), lastBackupFile: '', lastBackupTime: 0 - }, onCountdown); + }); + µb.assets.put(µb.userFiltersPath, userData.userFilters); + + // 'filterLists' is available up to uBO v1.10.4, not beyond. + // 'selectedFilterLists' is available from uBO v1.11 and beyond. + if ( Array.isArray(userData.selectedFilterLists) ) { + µb.saveSelectedFilterLists(userData.selectedFilterLists); + } else if ( userData.filterLists instanceof Object ) { + µb.saveSelectedFilterLists(µb.newListKeysFromOldData(userData.filterLists)); + } + + vAPI.app.restart(); }; // https://github.com/chrisaljoudi/uBlock/issues/1102 @@ -849,9 +849,7 @@ var prepListEntries = function(entries) { var µburi = µb.URI; var entry, hn; for ( var k in entries ) { - if ( entries.hasOwnProperty(k) === false ) { - continue; - } + if ( entries.hasOwnProperty(k) === false ) { continue; } entry = entries[k]; if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) { entry.supportName = µburi.hostnameFromURI(entry.supportURL); @@ -873,7 +871,8 @@ var getLists = function(callback) { current: µb.availableFilterLists, ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters, netFilterCount: µb.staticNetFilteringEngine.getFilterCount(), - userFiltersPath: µb.userFiltersPath + userFiltersPath: µb.userFiltersPath, + aliases: µb.assets.listKeyAliases }; var onMetadataReady = function(entries) { r.cache = entries; diff --git a/src/js/settings.js b/src/js/settings.js index 6e75aa1584e30..16582955916ed 100644 --- a/src/js/settings.js +++ b/src/js/settings.js @@ -56,7 +56,10 @@ var handleImportFilePicker = function() { if ( typeof userData.netWhitelist !== 'string' ) { throw 'Invalid'; } - if ( typeof userData.filterLists !== 'object' ) { + if ( + typeof userData.filterLists !== 'object' && + Array.isArray(userData.selectedFilterLists) === false + ) { throw 'Invalid'; } } diff --git a/src/js/storage.js b/src/js/storage.js index 9e29ec89ab9f1..6ca2b60cd256a 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -151,11 +151,27 @@ this.netWhitelistModifyTime = Date.now(); }; -/******************************************************************************/ +/******************************************************************************* + + TODO(seamless migration): + The code related to 'remoteBlacklist' can be removed when I am confident + all users have moved to a version of uBO which no longer depends on + the property 'remoteBlacklists, i.e. v1.11 and beyond. + +**/ µBlock.loadSelectedFilterLists = function(callback) { - vAPI.storage.get('selectedFilterLists', function(bin) { - callback(bin && bin.selectedFilterLists || undefined); + var µb = this; + vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists'], function(bin) { + if ( !bin ) { return callback(); } + if ( bin.selectedFilterLists ) { + return callback(bin.selectedFilterLists); + } + if ( !bin.remoteBlacklists ) { return callback(); } + var listKeys = µb.newListKeysFromOldData(bin.remoteBlacklists); + µb.saveSelectedFilterLists(listKeys); + vAPI.storage.remove('remoteBlacklists'); + callback(listKeys); }); }; @@ -169,6 +185,18 @@ } }; +µBlock.newListKeysFromOldData = function(oldLists) { + var aliases = this.assets.listKeyAliases, + listKeys = [], newKey; + for ( var oldKey in oldLists ) { + if ( oldLists[oldKey].off !== true ) { + newKey = aliases[oldKey]; + listKeys.push(newKey ? newKey : oldKey); + } + } + return listKeys; +}; + /******************************************************************************/ µBlock.saveUserFilters = function(content, callback) { @@ -187,9 +215,7 @@ /******************************************************************************/ µBlock.appendUserFilters = function(filters) { - if ( filters.length === 0 ) { - return; - } + if ( filters.length === 0 ) { return; } var µb = this; @@ -213,9 +239,7 @@ }; var onLoaded = function(details) { - if ( details.error ) { - return; - } + if ( details.error ) { return; } // https://github.com/chrisaljoudi/uBlock/issues/976 // If we reached this point, the filter quite probably needs to be // added for sure: do not try to be too smart, trying to avoid @@ -311,13 +335,13 @@ }; // Custom filter lists. - var listKeys = this.listKeysFromCustomFilterLists(µb.userSettings.externalLists), - i = listKeys.length, listKey, entry; + var importedListKeys = this.listKeysFromCustomFilterLists(µb.userSettings.externalLists), + i = importedListKeys.length, listKey, entry; while ( i-- ) { - listKey = listKeys[i]; + listKey = importedListKeys[i]; entry = { content: 'filters', - contentURL: listKeys[i], + contentURL: importedListKeys[i], external: true, group: 'custom', submitter: 'user', @@ -327,12 +351,14 @@ this.assets.registerAssetSource(listKey, entry); } - // Reuse existing list metadata if any. - var reuseMetadata = function() { - var assetKeys = Object.keys(oldAvailableLists), - assetKey, newEntry, oldEntry; + // Final steps: + // - reuse existing list metadata if any; + // - unregister unreferenced imported filter lists if any. + var finalize = function() { + var assetKey, newEntry, oldEntry; - while ( (assetKey = assetKeys.pop()) ) { + // Reuse existing metadata. + for ( assetKey in oldAvailableLists ) { oldEntry = oldAvailableLists[assetKey]; newEntry = newAvailableLists[assetKey]; if ( newEntry === undefined ) { @@ -358,6 +384,17 @@ newEntry.title = oldEntry.title; } } + + // Remove unreferenced imported filter lists. + var dict = new Set(importedListKeys); + for ( assetKey in newAvailableLists ) { + newEntry = newAvailableLists[assetKey]; + if ( newEntry.submitter !== 'user' ) { continue; } + if ( dict.has(assetKey) ) { continue; } + delete newAvailableLists[assetKey]; + µb.assets.unregisterAssetSource(assetKey); + µb.removeFilterList(assetKey); + } }; // Selected lists. @@ -375,18 +412,12 @@ µb.saveSelectedFilterLists(µb.autoSelectRegionalFilterLists(newAvailableLists)); } - reuseMetadata(); + finalize(); callback(newAvailableLists); }; // Built-in filter lists. - var onBuiltinListsLoaded = function(details) { - var entries, entry; - try { - entries = JSON.parse(details.content); - } catch (e) { - entries = {}; - } + var onBuiltinListsLoaded = function(entries) { for ( var assetKey in entries ) { if ( entries.hasOwnProperty(assetKey) === false ) { continue; } entry = entries[assetKey]; @@ -401,7 +432,7 @@ // Available lists previously computed. var onOldAvailableListsLoaded = function(bin) { oldAvailableLists = bin && bin.availableFilterLists || {}; - µb.assets.get('assets.json', onBuiltinListsLoaded); + µb.assets.metadata(onBuiltinListsLoaded); }; // Load previously saved available lists -- these contains data @@ -548,12 +579,11 @@ } } // Extract update frequency information - matches = head.match(/(?:^|\n)![\t ]*?Expires:[\t ]*?([\d]+)/i); + matches = head.match(/(?:^|\n)![\t ]*Expires:[\t ]*([\d]+)[\t ]*days?/i); if ( matches !== null ) { v = Math.max(parseInt(matches[1], 10), 2); if ( v !== listEntry.updateAfter ) { - listEntry.updateAfter = v; - this.assets.registerAssetSource(assetKey, listEntry); + this.assets.registerAssetSource(assetKey, { updateAfter: v }); } } }; @@ -817,8 +847,11 @@ } } - if ( typeof data.filterLists === 'object' ) { - bin.filterLists = data.filterLists; + if ( Array.isArray(data.selectedFilterLists) ) { + bin.selectedFilterLists = data.selectedFilterLists; + binNotEmpty = true; + } else if ( typeof data.filterLists === 'object' ) { + bin.selectedFilterLists = µb.newListKeysFromOldData(data.filterLists); binNotEmpty = true; } From 322560d92cb009cfd62527b3112e55a897347bf9 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 16 Jan 2017 13:37:11 -0500 Subject: [PATCH 03/15] various code review of new assets management code --- src/3p-filters.html | 7 +-- src/css/3p-filters.css | 31 +++++++---- src/js/3p-filters.js | 119 +++++++++++++++++++++++------------------ src/js/assets.js | 66 ++++++++++++++++------- src/js/messaging.js | 40 ++++++++++++-- 5 files changed, 175 insertions(+), 88 deletions(-) diff --git a/src/3p-filters.html b/src/3p-filters.html index faca189972f08..2cfd6b058d054 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -43,7 +43,7 @@ diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index 207318bff8240..8aa6760f9cf51 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -1,3 +1,7 @@ +@keyframes spin { + 100% { transform: rotate(360deg); -webkit-transform: rotate(360deg); } + } + ul { padding: 0; list-style-type: none; @@ -88,7 +92,7 @@ body[dir=rtl] #buttonApply { span.status { border: 1px solid transparent; color: #444; - display: inline-block; + display: none; font-size: smaller; line-height: 1; margin: 0 0 0 0.5em; @@ -98,30 +102,35 @@ span.status { span.unsecure { background-color: hsl(0, 100%, 88%); border-color: hsl(0, 100%, 83%); - display: none; } li.listEntry.unsecure span.unsecure { - display: unset; + display: inline; + } +span.obsolete { + background-color: hsl(36, 100%, 80%); + border-color: hsl(36, 100%, 75%); + } +li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete { + display: inline; } span.purge { border-color: #ddd; background-color: #eee; cursor: pointer; - display: none; } span.purge:hover { opacity: 1; } li.listEntry.cached span.purge { - display: unset; + display: inline; } -span.obsolete { - background-color: hsl(36, 100%, 80%); - border-color: hsl(36, 100%, 75%); - display: none; +span.updating { + border: none; + padding: 0; } -li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete { - display: unset; +li.listEntry.updating span.updating { + animation: spin 2s linear infinite; + display: inline-block; } #externalListsDiv { margin: 2em auto 0 2em; diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index e4339b8da91ce..8cdd39848e4bd 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -78,38 +78,40 @@ var renderFilterLists = function() { return listTitle; }; - var liFromListEntry = function(listKey) { + var liFromListEntry = function(listKey, li) { var entry = listDetails.available[listKey]; - var li = listEntryTemplate.clone(); - - li.attr('data-listkey', listKey); + li = li ? li : listEntryTemplate.clone().nodeAt(0); + li.setAttribute('data-listkey', listKey); + var elem = li.querySelector('input[type="checkbox"]'); if ( entry.off !== true ) { - li.descendants('input').attr('checked', ''); + elem.setAttribute('checked', ''); + } else { + elem.removeAttribute('checked'); } - - var elem = li.descendants('a:nth-of-type(1)'); - elem.attr('href', 'asset-viewer.html?url=' + encodeURI(listKey)); - elem.attr('type', 'text/html'); - elem.text(listNameFromListKey(listKey) + '\u200E'); - + elem = li.querySelector('a:nth-of-type(1)'); + elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); + elem.setAttribute('type', 'text/html'); + elem.textContent = listNameFromListKey(listKey) + '\u200E'; + elem = li.querySelector('a:nth-of-type(2)'); if ( entry.instructionURL ) { - elem = li.descendants('a:nth-of-type(2)'); - elem.attr('href', entry.instructionURL); - elem.css('display', ''); + elem.setAttribute('href', entry.instructionURL); + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); } - + elem = li.querySelector('a:nth-of-type(3)'); if ( entry.supportName ) { - elem = li.descendants('a:nth-of-type(3)'); - elem.attr('href', entry.supportURL); - elem.text('(' + entry.supportName + ')'); - elem.css('display', ''); + elem.setAttribute('href', entry.supportURL); + elem.textContent = '(' + entry.supportName + ')'; + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); } - - elem = li.descendants('span.counts'); + elem = li.querySelector('span.counts'); var text = listStatsTemplate .replace('{{used}}', renderNumber(!entry.off && !isNaN(+entry.entryUsedCount) ? entry.entryUsedCount : 0)) .replace('{{total}}', !isNaN(+entry.entryCount) ? renderNumber(entry.entryCount) : '?'); - elem.text(text); + elem.textContent = text; // https://github.com/chrisaljoudi/uBlock/issues/104 var asset = listDetails.cache[listKey] || {}; @@ -117,23 +119,27 @@ var renderFilterLists = function() { // https://github.com/gorhill/uBlock/issues/78 // Badge for non-secure connection var remoteURL = asset.remoteURL; - li.toggleClass( + li.classList.toggle( 'unsecure', typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0 ); - // Badge for update status - li.toggleClass('obsolete', entry.off !== true && asset.obsolete === true); - + li.classList.toggle( + 'obsolete', + entry.off !== true && asset.obsolete === true + ); // Badge for cache status - li.toggleClass('cached', asset.cached === true && asset.writeTime > 0); - + li.classList.toggle( + 'cached', + asset.cached === true && asset.writeTime > 0 + ); if ( asset.cached ) { - elem = li.descendants('.status.purge').attr( + li.querySelector('.status.purge').setAttribute( 'title', lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime)) ); } + li.classList.remove('discard'); return li; }; @@ -152,27 +158,29 @@ var renderFilterLists = function() { }; var liFromListGroup = function(groupKey, listKeys) { - var liGroup = listGroupTemplate.clone(); - var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1)); - if ( groupName !== '' ) { - liGroup.descendants('span.geName').text(groupName); - liGroup.descendants('span.geCount').text(listEntryCountFromGroup(listKeys)); - } - var ulGroup = liGroup.descendants('ul'); - if ( !listKeys ) { - return liGroup; + var liGroup = document.querySelector('#lists > .groupEntry[data-groupkey="' + groupKey + '"]'); + if ( liGroup === null ) { + liGroup = listGroupTemplate.clone().nodeAt(0); + var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1)); + if ( groupName !== '' ) { + liGroup.querySelector('.geName').textContent = groupName; + liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys); + } } + var ulGroup = liGroup.querySelector('.listEntries'); + if ( !listKeys ) { return liGroup; } listKeys.sort(function(a, b) { return (listDetails.available[a].title || '').localeCompare(listDetails.available[b].title || ''); }); for ( var i = 0; i < listKeys.length; i++ ) { - ulGroup.append(liFromListEntry(listKeys[i])); + var liEntry = liFromListEntry(listKeys[i], ulGroup.children[i]); + if ( liEntry.parentElement === null ) { + ulGroup.appendChild(liEntry); + } } return liGroup; }; - // https://www.youtube.com/watch?v=unCVi4hYRlY#t=30m18s - var groupsFromLists = function(lists) { var groups = {}; var listKeys = Object.keys(lists); @@ -196,11 +204,15 @@ var renderFilterLists = function() { parseCosmeticFilters = details.parseCosmeticFilters; ignoreGenericCosmeticFilters = details.ignoreGenericCosmeticFilters; + // Incremental rendering: this will allow us to easily discard unused + // DOM list entries. + uDom('#lists .listEntries .listEntry').addClass('discard'); + // Visually split the filter lists in purpose-based groups - var ulLists = uDom('#lists').empty(), liGroup; - var groups = groupsFromLists(details.available); - var groupKey, i; - var groupKeys = [ + var ulLists = document.querySelector('#lists'), + groups = groupsFromLists(details.available), + liGroup, i, groupKey, + groupKeys = [ 'default', 'ads', 'privacy', @@ -213,20 +225,24 @@ var renderFilterLists = function() { for ( i = 0; i < groupKeys.length; i++ ) { groupKey = groupKeys[i]; liGroup = liFromListGroup(groupKey, groups[groupKey]); - liGroup.toggleClass( + liGroup.setAttribute('data-groupkey', groupKey); + liGroup.classList.toggle( 'collapsed', vAPI.localStorage.getItem('collapseGroup' + (i + 1)) === 'y' ); - ulLists.append(liGroup); + if ( liGroup.parentElement === null ) { + ulLists.appendChild(liGroup); + } delete groups[groupKey]; } // For all groups not covered above (if any left) groupKeys = Object.keys(groups); for ( i = 0; i < groupKeys.length; i++ ) { groupKey = groupKeys[i]; - ulLists.append(liFromListGroup(groupKey, groups[groupKey])); + ulLists.appendChild(liFromListGroup(groupKey, groups[groupKey])); } + uDom('#lists .listEntries .listEntry.discard').remove(); uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); @@ -259,6 +275,7 @@ var renderFilterLists = function() { var renderWidgets = function() { uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); }; /******************************************************************************/ @@ -267,6 +284,8 @@ var updateAssetStatus = function(details) { var li = uDom('#lists .listEntry[data-listkey="' + details.key + '"]'); li.toggleClass('obsolete', !details.cached); li.toggleClass('cached', details.cached); + li.removeClass('updating'); + renderWidgets(); }; /******************************************************************************/ @@ -305,7 +324,6 @@ var onPurgeClicked = function() { if ( !listKey ) { return; } messaging.send('dashboard', { what: 'purgeCache', path: listKey }); - button.remove(); // If the cached version is purged, the installed version must be assumed // to be obsolete. @@ -373,13 +391,12 @@ var buttonApplyHandler = function() { /******************************************************************************/ var buttonUpdateHandler = function(ev) { - uDom('#buttonUpdate').removeClass('enabled'); var onSelectionDone = function() { + uDom('#lists .listEntry.obsolete').addClass('updating'); messaging.send('dashboard', { what: 'forceUpdateAssets', fast: ev.ctrlKey }); - uDom('#buttonUpdate').addClass('disabled'); }; selectFilterLists(onSelectionDone); }; diff --git a/src/js/assets.js b/src/js/assets.js index 34e661e2048dd..a9039f3b5ea5d 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -313,6 +313,7 @@ var registerAssetSource = function(assetKey, dict) { }; var unregisterAssetSource = function(assetKey) { + assetCacheRemove(assetKey); delete assetSourceRegistry[assetKey]; }; @@ -334,6 +335,30 @@ var saveAssetSourceRegistry = (function() { }; })(); +var updateAssetSourceRegistry = function(json) { + var newDict; + try { + newDict = JSON.parse(json); + } catch (ex) { + } + if ( newDict instanceof Object === false ) { return; } + + getAssetSourceRegistry(function(oldDict) { + var assetKey; + // Remove obsolete entries + for ( assetKey in oldDict ) { + if ( newDict[assetKey] === undefined ) { + unregisterAssetSource(assetKey); + } + } + // Add/update existing entries + for ( assetKey in newDict ) { + registerAssetSource(assetKey, newDict[assetKey]); + } + saveAssetSourceRegistry(); + }); +}; + var getAssetSourceRegistry = function(callback) { // Already loaded. if ( assetSourceRegistryStatus === 'ready' ) { @@ -362,15 +387,7 @@ var getAssetSourceRegistry = function(callback) { // First-install case. var createRegistry = function() { getTextFileFromURL(vAPI.getURL('assets/assets.json'), function() { - var assetDict; - try { - assetDict = JSON.parse(this.responseText); - } catch (ex) { - } - for ( var assetKey in assetDict ) { - registerAssetSource(assetKey, assetDict[assetKey]); - } - saveAssetSourceRegistry(); + updateAssetSourceRegistry(this.responseText); registryReady(); }); }; @@ -409,6 +426,7 @@ api.unregisterAssetSource = function(assetKey) { **/ var assetCacheRegistryStatus, + assetCacheRegistryStartTime = Date.now(), assetCacheRegistry = {}; var getAssetCacheRegistry = function(callback) { @@ -856,20 +874,18 @@ api.rmrf = function() { var updateStatus; var updateTimer; -var updateAssetDelay = 4000; +var updateAssetDelayDefault = 2 * 60 * 1000; +var updateAssetDelay = updateAssetDelayDefault; var updated = []; var updateFirst = function() { updateStatus = 'updating'; updated = []; - fireNotification('before-assets-updated'); - updateNext(); }; var updateNext = function() { - var assetDict, cacheDict; var findOne = function() { @@ -882,20 +898,25 @@ var updateNext = function() { if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { continue; } - if ( assetEntry.error ) { - if ( assetEntry.error.time + 3600000 > now ) { - continue; - } + if ( assetEntry.error && assetEntry.error.time + 3600000 > now ) { + continue; } if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) { return assetKey; } + // This will remove the cached asset if it's no longer in use. + if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { + assetCacheRemove(assetKey); + } } }; var updatedOne = function(details) { if ( details.content !== '' ) { updated.push(details.assetKey); + if ( details.assetKey === 'assets.json' ) { + updateAssetSourceRegistry(details.content); + } } if ( findOne() !== undefined ) { vAPI.setTimeout(updateNext, updateAssetDelay); @@ -929,14 +950,19 @@ var updateDone = function() { var assetKeys = updated.slice(0); updated = []; updateStatus = undefined; + updateAssetDelay = updateAssetDelayDefault; fireNotification('after-assets-updated', { assetKeys: assetKeys }); }; api.updateStart = function(details) { - updateAssetDelay = details.delay; + var oldUpdateDelay = updateAssetDelay, + newUpdateDelay = details.delay || updateAssetDelayDefault; + updateAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); if ( updateStatus !== undefined ) { - clearTimeout(updateTimer); - updateTimer = vAPI.setTimeout(updateNext, updateAssetDelay); + if ( newUpdateDelay < oldUpdateDelay ) { + clearTimeout(updateTimer); + updateTimer = vAPI.setTimeout(updateNext, updateAssetDelay); + } return; } updateFirst(); diff --git a/src/js/messaging.js b/src/js/messaging.js index 6b0970fb9a286..c73de7f161a4b 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -122,7 +122,7 @@ var onMessage = function(request, sender, callback) { case 'forceUpdateAssets': µb.scheduleAssetUpdater(0); - µb.assets.updateStart({ delay: request.fast ? 1 : 3 * 1000 }); + µb.assets.updateStart({ delay: request.fast ? 1 : 2000 }); break; case 'getAppData': @@ -766,6 +766,42 @@ var backupUserData = function(callback) { var onSelectedListsReady = function(selectedFilterLists) { userData.selectedFilterLists = selectedFilterLists; + // TODO(seamless migration): + // The following is strictly for convenience, to be minimally + // forward-compatible. This will definitely be removed in the + // short term, as I do not expect the need to install an older + // version of uBO to ever be needed beyond the short term. + // >>>>>>>> + var reverseAliases = Object.keys(µb.assets.listKeyAliases).reduce( + function(a, b) { + a[µb.assets.listKeyAliases[b]] = b; return a; + }, + {} + ); + userData.filterLists = selectedFilterLists.reduce( + function(a, b) { + a[reverseAliases[b] || b] = { off: false }; + return a; + }, + {} + ); + userData.filterLists = Object.keys(µb.assets.listKeyAliases).reduce( + function(a, b) { + var aliases = µb.assets.listKeyAliases; + if ( + b.startsWith('assets/') && + aliases[b] !== 'public_suffix_list.dat' && + aliases[b] !== 'ublock-resources' && + !a[b] + ) { + a[b] = { off: true }; + } + return a; + }, + userData.filterLists + ); + // <<<<<<<< + var filename = vAPI.i18n('aboutBackupFilename') .replace('{{datetime}}', µb.dateNowToSensibleString()) .replace(/ +/g, '_'); @@ -774,11 +810,9 @@ var backupUserData = function(callback) { 'url': 'data:text/plain;charset=utf-8,' + encodeURIComponent(JSON.stringify(userData, null, ' ')), 'filename': filename }); - µb.restoreBackupSettings.lastBackupFile = filename; µb.restoreBackupSettings.lastBackupTime = Date.now(); vAPI.storage.set(µb.restoreBackupSettings); - getLocalData(callback); }; From 3ec7fc5cbedf56335aa831d7cacac992c0f15c7a Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 16 Jan 2017 13:42:24 -0500 Subject: [PATCH 04/15] fix #2281 --- assets/assets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/assets.json b/assets/assets.json index 6ba6af58933e0..193156c32188a 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -202,7 +202,7 @@ "off": true, "title": "ITA: ABP X Files", "group": "regions", - "contentURL": "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt", + "contentURL": "https://raw.githubusercontent.com/gioxx/xfiles/master/filtri.txt", "supportURL": "http://noads.it/" }, "RUS-0": { From ca9e12ce672a55040a54805ae789264f35247b8a Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 16 Jan 2017 13:46:50 -0500 Subject: [PATCH 05/15] fix #1961 --- assets/assets.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/assets.json b/assets/assets.json index 193156c32188a..6877a7bfbe916 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -479,8 +479,8 @@ "title": "BGR: Bulgarian Adblock list", "group": "regions", "lang": "bg", - "contentURL": "http://stanev.org/abp/adblock_bg.txt", - "supportURL": "http://stanev.org/abp/" + "contentURL": "https://stanev.org/abp/adblock_bg.txt", + "supportURL": "https://stanev.org/abp/" }, "mvps-0": { "content": "filters", From d32bcbda0748057b66f5fecf6b63c4e208654cfb Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 16 Jan 2017 13:54:52 -0500 Subject: [PATCH 06/15] fix #1293 --- assets/assets.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/assets/assets.json b/assets/assets.json index 6877a7bfbe916..f5d1c5c5c472f 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -344,8 +344,11 @@ "title": "IDN: ABPindo", "group": "regions", "lang": "id", - "contentURL": "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt", - "supportURL": "https://github.com/heradhis/indonesianadblockrules" + "contentURL": [ + "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt" + ], + "supportURL": "https://github.com/ABPindo/indonesianadblockrules" }, "JPN-0": { "content": "filters", From 30aef303e598ea298a50fecc7d29c5f72ae86fb0 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 16 Jan 2017 13:57:58 -0500 Subject: [PATCH 07/15] fix #1275 --- assets/assets.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/assets.json b/assets/assets.json index f5d1c5c5c472f..3042d1c9565a5 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -292,8 +292,8 @@ "title": "POL: polskie filtry do Adblocka i uBlocka", "group": "regions", "lang": "pl", - "contentURL": "https://www.certyficate.it/adblock/adblock.txt", - "supportURL": "http://www.certyficate.it/adblock-ublock-polish-filters/" + "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", + "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/" }, "awrl-0": { "content": "filters", From 054a60b7d593cc9e20ba811ad07afb0c4924ad76 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 16 Jan 2017 17:18:46 -0500 Subject: [PATCH 08/15] fix update scheduler timing logic --- src/js/storage.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/js/storage.js b/src/js/storage.js index 6ca2b60cd256a..e9e177c5f728c 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -897,14 +897,16 @@ if ( timer ) { clearTimeout(timer); timer = undefined; + } + if ( updateDelay === 0 ) { next = 0; + return; } - if ( updateDelay === 0 ) { return; } var now = Date.now(); // Use the new schedule if and only if it is earlier than the previous // one. if ( next !== 0 ) { - updateDelay = Math.min(updateDelay, next - now); + updateDelay = Math.min(updateDelay, Math.max(next - now, 0)); } next = now + updateDelay; timer = vAPI.setTimeout(function() { @@ -970,7 +972,7 @@ if ( details.assetKeys.length !== 0 ) { this.loadFilterLists(); } - this.scheduleAssetUpdater(this.userSettings.autoUpdate ? 11 * 60 * 60 * 1000 : -1); + this.scheduleAssetUpdater(this.userSettings.autoUpdate ? 11 * 60 * 60 * 1000 : 0); return; } }; From 046a1af60a57cb0910edacf5c6e2ee4aaafd5f9e Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 17 Jan 2017 11:14:41 -0500 Subject: [PATCH 09/15] forward compatibility (to be removed once 1.11+ is widespread) --- src/js/assets.js | 48 ++++++++++++++++++++++--------- src/js/messaging.js | 45 +++++++---------------------- src/js/storage.js | 70 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 107 insertions(+), 56 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index a9039f3b5ea5d..3dae5bc01557e 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -236,6 +236,9 @@ var migrate = function(callback) { var onEntries = function(bin) { entries = bin && bin['cached_asset_entries']; if ( !entries ) { return callback(); } + if ( bin && bin['assetCacheRegistry'] ) { + assetCacheRegistry = bin['assetCacheRegistry']; + } var aliases = api.listKeyAliases; for ( var oldKey in entries ) { if ( oldKey.endsWith('assets/user/filters.txt') ) { continue; } @@ -256,7 +259,10 @@ var migrate = function(callback) { countdown(); }; - vAPI.cacheStorage.get('cached_asset_entries', onEntries); + vAPI.cacheStorage.get( + [ 'cached_asset_entries', 'assetCacheRegistry' ], + onEntries + ); }; /******************************************************************************* @@ -304,7 +310,7 @@ var registerAssetSource = function(assetKey, dict) { entry.contentURL = []; } if ( typeof entry.updateAfter !== 'number' ) { - entry.updateAfter = 13; + entry.updateAfter = 5; } if ( entry.submitter ) { entry.submitTime = Date.now(); // To detect stale entries @@ -454,14 +460,16 @@ var getAssetCacheRegistry = function(callback) { } }; - vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { - if ( bin && bin.assetCacheRegistry ) { - assetCacheRegistry = bin.assetCacheRegistry; + var migrationDone = function() { + vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { + if ( bin && bin.assetCacheRegistry ) { + assetCacheRegistry = bin.assetCacheRegistry; + } registryReady(); - } else { - migrate(registryReady); - } - }); + }); + }; + + migrate(migrationDone); }; var saveAssetCacheRegistry = (function() { @@ -639,14 +647,18 @@ var readUserAsset = function(assetKey, callback) { } if ( typeof bin['assets/user/filters.txt'] === 'string' ) { content = bin['assets/user/filters.txt']; - vAPI.storage.remove('assets/user/filters.txt'); + // TODO(seamless migration): + // Uncomment once all moved to v1.11+. + //vAPI.storage.remove('assets/user/filters.txt'); } if ( typeof bin[assetKey] === 'string' ) { - content = bin[assetKey]; + // TODO(seamless migration): + // Replace conditional with assignment once all moved to v1.11+ + if ( content !== bin[assetKey] ) { + saveUserAsset(assetKey, content); + } } else if ( content !== '' ) { - bin = {}; - bin[assetKey] = content; - vAPI.storage.set(bin); + saveUserAsset(assetKey, content); } return reportBack(content); }; @@ -664,6 +676,14 @@ var readUserAsset = function(assetKey, callback) { var saveUserAsset = function(assetKey, content, callback) { var bin = {}; bin[assetKey] = content; + // TODO(seamless migration): + // This is for forward compatibility. Only for a limited time. Remove when + // everybody moved to 1.11.0 and beyond. + // >>>>>>>> + if ( assetKey === µBlock.userFiltersPath ) { + bin['assets/user/filters.txt'] = content; + } + // <<<<<<<< var onSaved = function() { if ( callback instanceof Function ) { callback({ assetKey: assetKey, content: content }); diff --git a/src/js/messaging.js b/src/js/messaging.js index c73de7f161a4b..dfcf92c5a7ff2 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -772,34 +772,7 @@ var backupUserData = function(callback) { // short term, as I do not expect the need to install an older // version of uBO to ever be needed beyond the short term. // >>>>>>>> - var reverseAliases = Object.keys(µb.assets.listKeyAliases).reduce( - function(a, b) { - a[µb.assets.listKeyAliases[b]] = b; return a; - }, - {} - ); - userData.filterLists = selectedFilterLists.reduce( - function(a, b) { - a[reverseAliases[b] || b] = { off: false }; - return a; - }, - {} - ); - userData.filterLists = Object.keys(µb.assets.listKeyAliases).reduce( - function(a, b) { - var aliases = µb.assets.listKeyAliases; - if ( - b.startsWith('assets/') && - aliases[b] !== 'public_suffix_list.dat' && - aliases[b] !== 'ublock-resources' && - !a[b] - ) { - a[b] = { off: true }; - } - return a; - }, - userData.filterLists - ); + userData.filterLists = µb.oldDataFromNewListKeys(selectedFilterLists); // <<<<<<<< var filename = vAPI.i18n('aboutBackupFilename') @@ -983,12 +956,6 @@ var onMessage = function(request, sender, callback) { case 'getLocalData': return getLocalData(callback); - case 'purgeAllCaches': - if ( request.hard ) { - return µb.assets.remove(/./, callback); - } - return µb.assets.purge(/./, callback); - case 'readUserFilters': return µb.loadUserFilters(callback); @@ -1007,8 +974,18 @@ var onMessage = function(request, sender, callback) { response = getRules(); break; + case 'purgeAllCaches': + if ( request.hard ) { + µb.assets.remove(/./); + } else { + µb.assets.remove(/compiled\//); + µb.assets.purge(/./); + } + break; + case 'purgeCache': µb.assets.purge(request.path); + µb.assets.remove('compiled/' + request.path); break; case 'readHiddenSettings': diff --git a/src/js/storage.js b/src/js/storage.js index e9e177c5f728c..5e15c7a77d344 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -162,29 +162,48 @@ µBlock.loadSelectedFilterLists = function(callback) { var µb = this; - vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists'], function(bin) { + vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists' ], function(bin) { if ( !bin ) { return callback(); } + var listKeys = []; if ( bin.selectedFilterLists ) { - return callback(bin.selectedFilterLists); + listKeys = bin.selectedFilterLists; + } + if ( bin.remoteBlacklists ) { + var oldListKeys = µb.newListKeysFromOldData(bin.remoteBlacklists); + if ( oldListKeys.sort().join() !== listKeys.sort().join() ) { + listKeys = oldListKeys; + µb.saveSelectedFilterLists(listKeys); + } + // TODO(seamless migration): + // Uncomment when all have moved to v1.11 and beyond. + //vAPI.storage.remove('remoteBlacklists'); } - if ( !bin.remoteBlacklists ) { return callback(); } - var listKeys = µb.newListKeysFromOldData(bin.remoteBlacklists); - µb.saveSelectedFilterLists(listKeys); - vAPI.storage.remove('remoteBlacklists'); callback(listKeys); }); }; µBlock.saveSelectedFilterLists = function(listKeys, append) { + var µb = this; + var save = function(keys) { + var bin = { + selectedFilterLists: keys, + remoteBlacklists: µb.oldDataFromNewListKeys(keys) + }; + vAPI.storage.set(bin); + }; if ( append ) { this.loadSelectedFilterLists(function(keys) { - vAPI.storage.set({ selectedFilterLists: listKeys.concat(keys || []) }); + listKeys = listKeys.concat(keys || []); + save(listKeys); }); } else { - vAPI.storage.set({ selectedFilterLists: listKeys }); + save(listKeys); } }; +// TODO(seamless migration): +// Remove when all have moved to v1.11 and beyond. +// >>>>>>>> µBlock.newListKeysFromOldData = function(oldLists) { var aliases = this.assets.listKeyAliases, listKeys = [], newKey; @@ -197,6 +216,41 @@ return listKeys; }; +µBlock.oldDataFromNewListKeys = function(selectedFilterLists) { + var µb = this, + remoteBlacklists = {}; + var reverseAliases = Object.keys(this.assets.listKeyAliases).reduce( + function(a, b) { + a[µb.assets.listKeyAliases[b]] = b; return a; + }, + {} + ); + remoteBlacklists = selectedFilterLists.reduce( + function(a, b) { + a[reverseAliases[b] || b] = { off: false }; + return a; + }, + {} + ); + remoteBlacklists = Object.keys(µb.assets.listKeyAliases).reduce( + function(a, b) { + var aliases = µb.assets.listKeyAliases; + if ( + b.startsWith('assets/') && + aliases[b] !== 'public_suffix_list.dat' && + aliases[b] !== 'ublock-resources' && + !a[b] + ) { + a[b] = { off: true }; + } + return a; + }, + remoteBlacklists + ); + return remoteBlacklists; +}; +// <<<<<<<< + /******************************************************************************/ µBlock.saveUserFilters = function(content, callback) { From 41ff9694b10589d5f59dcc2dd3fd3c72b49b1ffb Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 17 Jan 2017 13:13:46 -0500 Subject: [PATCH 10/15] more codereview; give admins ability to specify own assets.json --- src/js/assets.js | 90 +++++++++++++++++++++++--------------------- src/js/background.js | 12 +++--- src/js/storage.js | 13 ++++++- 3 files changed, 65 insertions(+), 50 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index 3dae5bc01557e..fb86926ac51a3 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -65,7 +65,9 @@ var fireNotification = function(topic, details) { /******************************************************************************/ var getTextFileFromURL = function(url, onLoad, onError) { - // console.log('µBlock.assets/getTextFileFromURL("%s"):', url); + if ( reIsExternalPath.test(url) === false ) { + url = vAPI.getURL(url); + } if ( typeof onError !== 'function' ) { onError = onLoad; @@ -392,10 +394,13 @@ var getAssetSourceRegistry = function(callback) { // First-install case. var createRegistry = function() { - getTextFileFromURL(vAPI.getURL('assets/assets.json'), function() { - updateAssetSourceRegistry(this.responseText); - registryReady(); - }); + getTextFileFromURL( + µBlock.assetsBootstrapLocation || 'assets/assets.json', + function() { + updateAssetSourceRegistry(this.responseText); + registryReady(); + } + ); }; vAPI.cacheStorage.get('assetSourceRegistry', function(bin) { @@ -408,8 +413,6 @@ var getAssetSourceRegistry = function(callback) { }); }; -//var updateAssetSourceRegistry = function(raw) {}; - api.registerAssetSource = function(assetKey, details) { getAssetSourceRegistry(function() { registerAssetSource(assetKey, details); @@ -725,11 +728,7 @@ api.get = function(assetKey, callback) { if ( !contentURL ) { return reportBack('', 'E_NOTFOUND'); } - getTextFileFromURL( - isExternal ? contentURL : vAPI.getURL(contentURL), - onContentLoaded, - onContentNotLoaded - ); + getTextFileFromURL(contentURL, onContentLoaded, onContentNotLoaded); }; var onContentLoaded = function() { @@ -892,15 +891,17 @@ api.rmrf = function() { // Asset updater area. -var updateStatus; -var updateTimer; -var updateAssetDelayDefault = 2 * 60 * 1000; -var updateAssetDelay = updateAssetDelayDefault; -var updated = []; +var updaterStatus; +var updaterTimer; +var updaterAssetDelayDefault = 2 * 60 * 1000; +var updaterAssetDelay = updaterAssetDelayDefault; +var updaterUpdated = []; +var updaterFetched = new Set(); var updateFirst = function() { - updateStatus = 'updating'; - updated = []; + updaterStatus = 'updating'; + updaterFetched.clear(); + updaterUpdated = []; fireNotification('before-assets-updated'); updateNext(); }; @@ -908,38 +909,41 @@ var updateFirst = function() { var updateNext = function() { var assetDict, cacheDict; + // This will remove a cached asset when it's no longer in use. + var garbageCollectOne = function(assetKey) { + var cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { + assetCacheRemove(assetKey); + } + }; + var findOne = function() { var now = Date.now(), assetEntry, cacheEntry; for ( var assetKey in assetDict ) { assetEntry = assetDict[assetKey]; if ( assetEntry.hasRemoteURL !== true ) { continue; } + if ( updaterFetched.has(assetKey) ) { continue; } cacheEntry = cacheDict[assetKey]; if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { continue; } - if ( assetEntry.error && assetEntry.error.time + 3600000 > now ) { - continue; - } if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) { return assetKey; } - // This will remove the cached asset if it's no longer in use. - if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { - assetCacheRemove(assetKey); - } + garbageCollectOne(assetKey); } }; var updatedOne = function(details) { if ( details.content !== '' ) { - updated.push(details.assetKey); + updaterUpdated.push(details.assetKey); if ( details.assetKey === 'assets.json' ) { updateAssetSourceRegistry(details.content); } } if ( findOne() !== undefined ) { - vAPI.setTimeout(updateNext, updateAssetDelay); + vAPI.setTimeout(updateNext, updaterAssetDelay); } else { updateDone(); } @@ -950,6 +954,7 @@ var updateNext = function() { if ( assetKey === undefined ) { return updateDone(); } + updaterFetched.add(assetKey); getRemote(assetKey, updatedOne); }; @@ -967,21 +972,22 @@ var updateNext = function() { }; var updateDone = function() { - var assetKeys = updated.slice(0); - updated = []; - updateStatus = undefined; - updateAssetDelay = updateAssetDelayDefault; + var assetKeys = updaterUpdated.slice(0); + updaterFetched.clear(); + updaterUpdated = []; + updaterStatus = undefined; + updaterAssetDelay = updaterAssetDelayDefault; fireNotification('after-assets-updated', { assetKeys: assetKeys }); }; api.updateStart = function(details) { - var oldUpdateDelay = updateAssetDelay, - newUpdateDelay = details.delay || updateAssetDelayDefault; - updateAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); - if ( updateStatus !== undefined ) { + var oldUpdateDelay = updaterAssetDelay, + newUpdateDelay = details.delay || updaterAssetDelayDefault; + updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); + if ( updaterStatus !== undefined ) { if ( newUpdateDelay < oldUpdateDelay ) { - clearTimeout(updateTimer); - updateTimer = vAPI.setTimeout(updateNext, updateAssetDelay); + clearTimeout(updaterTimer); + updaterTimer = vAPI.setTimeout(updateNext, updaterAssetDelay); } return; } @@ -989,11 +995,11 @@ api.updateStart = function(details) { }; api.updateStop = function() { - if ( updateTimer ) { - clearTimeout(updateTimer); - updateTimer = undefined; + if ( updaterTimer ) { + clearTimeout(updaterTimer); + updaterTimer = undefined; } - if ( updateStatus !== undefined ) { + if ( updaterStatus !== undefined ) { updateDone(); } }; diff --git a/src/js/background.js b/src/js/background.js index bb7d462120473..c82fa051e05db 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -29,8 +29,6 @@ var µBlock = (function() { // jshint ignore:line var oneSecond = 1000; var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -// var oneDay = 24 * oneHour; /******************************************************************************/ @@ -117,14 +115,14 @@ return { lastBackupTime: 0 }, - // EasyList, EasyPrivacy and many others have an 4-day update period, - // as per list headers. - updateAssetsEvery: 97 * oneHour, - projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/', + // Allows to fully customize uBO's assets, typically set through admin + // settings. The content of 'assets.json' will also tell which filter + // lists to enable by default when uBO is first installed. + assetsBootstrapLocation: 'assets/assets.json', + userFiltersPath: 'user-filters', pslAssetKey: 'public_suffix_list.dat', - // current lists availableFilterLists: {}, selfieAfter: 23 * oneMinute, diff --git a/src/js/storage.js b/src/js/storage.js index 5e15c7a77d344..43421bcf8dff6 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -163,7 +163,9 @@ µBlock.loadSelectedFilterLists = function(callback) { var µb = this; vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists' ], function(bin) { - if ( !bin ) { return callback(); } + if ( !bin || !bin.selectedFilterLists && !bin.remoteBlacklists ) { + return callback(); + } var listKeys = []; if ( bin.selectedFilterLists ) { listKeys = bin.selectedFilterLists; @@ -888,6 +890,13 @@ var bin = {}; var binNotEmpty = false; + // Allows an admin to set their own 'assets.json' file, with their own + // set of stock assets. + if ( typeof data.assetsBootstrapLocation === 'string' ) { + bin.assetsBootstrapLocation = data.assetsBootstrapLocation; + binNotEmpty = true; + } + if ( typeof data.userSettings === 'object' ) { for ( var name in µb.userSettings ) { if ( µb.userSettings.hasOwnProperty(name) === false ) { @@ -901,6 +910,8 @@ } } + // 'selectedFilterLists' is an array of filter list tokens. Each token + // is a reference to an asset in 'assets.json'. if ( Array.isArray(data.selectedFilterLists) ) { bin.selectedFilterLists = data.selectedFilterLists; binNotEmpty = true; From 3d5c3ef68f0d95de4cb764397ca61ccf006a28f0 Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 17 Jan 2017 15:42:20 -0500 Subject: [PATCH 11/15] "assetKey" is more accurate than "path" --- src/js/3p-filters.js | 2 +- src/js/messaging.js | 4 ++-- src/js/reverselookup-worker.js | 18 +++++++++--------- src/js/reverselookup.js | 8 ++++---- src/js/start.js | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 8cdd39848e4bd..77cec9dbab3df 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -323,7 +323,7 @@ var onPurgeClicked = function() { listKey = liEntry.attr('data-listkey'); if ( !listKey ) { return; } - messaging.send('dashboard', { what: 'purgeCache', path: listKey }); + messaging.send('dashboard', { what: 'purgeCache', assetKey: listKey }); // If the cached version is purged, the installed version must be assumed // to be obsolete. diff --git a/src/js/messaging.js b/src/js/messaging.js index dfcf92c5a7ff2..6ab60da30976e 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -984,8 +984,8 @@ var onMessage = function(request, sender, callback) { break; case 'purgeCache': - µb.assets.purge(request.path); - µb.assets.remove('compiled/' + request.path); + µb.assets.purge(request.assetKey); + µb.assets.remove('compiled/' + request.assetKey); break; case 'readHiddenSettings': diff --git a/src/js/reverselookup-worker.js b/src/js/reverselookup-worker.js index f52002e676750..17e8f99151af5 100644 --- a/src/js/reverselookup-worker.js +++ b/src/js/reverselookup-worker.js @@ -1,7 +1,7 @@ /******************************************************************************* - uBlock - a browser extension to block requests. - Copyright (C) 2015 Raymond Hill + uBlock Origin - a browser extension to block requests. + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -43,8 +43,8 @@ var fromNetFilter = function(details) { var lists = []; var compiledFilter = details.compiledFilter; var entry, content, pos, c; - for ( var path in listEntries ) { - entry = listEntries[path]; + for ( var assetKey in listEntries ) { + entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } @@ -173,11 +173,11 @@ var fromCosmeticFilter = function(details) { ); } - var re, path, entry; + var re, assetKey, entry; for ( var candidate in candidates ) { re = candidates[candidate]; - for ( path in listEntries ) { - entry = listEntries[path]; + for ( assetKey in listEntries ) { + entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } @@ -206,7 +206,7 @@ var reHighMedium = /^\[href\^="https?:\/\/([^"]{8})[^"]*"\]$/; /******************************************************************************/ -onmessage = function(e) { +onmessage = function(e) { // jshint ignore:line var msg = e.data; switch ( msg.what ) { @@ -215,7 +215,7 @@ onmessage = function(e) { break; case 'setList': - listEntries[msg.details.path] = msg.details; + listEntries[msg.details.assetKey] = msg.details; break; case 'fromNetFilter': diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index f2a5961121a0d..c18bfba6662b7 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -75,16 +75,16 @@ var initWorker = function(callback) { var countdown = 0; var onListLoaded = function(details) { - var entry = entries[details.path]; + var entry = entries[details.assetKey]; // https://github.com/gorhill/uBlock/issues/536 - // Use path string when there is no filter list title. + // Use assetKey when there is no filter list title. worker.postMessage({ what: 'setList', details: { - path: details.path, - title: entry.title || details.path, + assetKey: details.assetKey, + title: entry.title || details.assetKey, supportURL: entry.supportURL, content: details.content } diff --git a/src/js/start.js b/src/js/start.js index 97a80408310bf..2ccd4fc8dc061 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by From 31555ca86e073a097d79255da4c7327be56201ec Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 17 Jan 2017 15:46:21 -0500 Subject: [PATCH 12/15] fix group count update when building dom incrementally --- src/js/3p-filters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 77cec9dbab3df..5f307f14e0fa0 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -164,9 +164,9 @@ var renderFilterLists = function() { var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1)); if ( groupName !== '' ) { liGroup.querySelector('.geName').textContent = groupName; - liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys); } } + liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys); var ulGroup = liGroup.querySelector('.listEntries'); if ( !listKeys ) { return liGroup; } listKeys.sort(function(a, b) { From dfb2a72c9106a3a286e432d5af0962de53763912 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 08:03:07 -0500 Subject: [PATCH 13/15] reorganize content (order, added URLs, etc.) --- assets/assets.json | 635 +++++++++++++++++++++++---------------------- 1 file changed, 323 insertions(+), 312 deletions(-) diff --git a/assets/assets.json b/assets/assets.json index 3042d1c9565a5..883b87ba01ee4 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -25,35 +25,17 @@ }, "ublock-filters": { "content": "filters", - "title": "uBlock filters", "group": "default", + "title": "uBlock filters", "contentURL": [ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt", "assets/ublock/filters.txt" ] }, - "ublock-privacy": { - "content": "filters", - "title": "uBlock filters – Privacy", - "group": "default", - "contentURL": [ - "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt", - "assets/ublock/privacy.txt" - ] - }, - "ublock-unbreak": { - "content": "filters", - "title": "uBlock filters – Unbreak", - "group": "default", - "contentURL": [ - "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt", - "assets/ublock/unbreak.txt" - ] - }, "ublock-badware": { "content": "filters", - "title": "uBlock filters – Badware risks", "group": "default", + "title": "uBlock filters – Badware risks", "contentURL": [ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt", "assets/ublock/badware.txt" @@ -63,8 +45,8 @@ }, "ublock-experimental": { "content": "filters", - "title": "uBlock filters – Experimental", "group": "default", + "title": "uBlock filters – Experimental", "off": true, "contentURL": [ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/experimental.txt", @@ -73,53 +55,45 @@ "supportURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters", "instructionURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters" }, - "CHN-0": { - "content": "filters", - "off": true, - "title": "CHN: EasyList China (中文)", - "group": "regions", - "lang": "zh", - "contentURL": "https://easylist-downloads.adblockplus.org/easylistchina.txt", - "supportURL": "http://abpchina.org/forum/forum.php" - }, - "CHN-1": { + "ublock-privacy": { "content": "filters", - "off": true, - "title": "CHN: CJX's EasyList Lite (main focus on Chinese sites)", - "group": "regions", - "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", - "supportURL": "https://github.com/cjx82630/cjxlist" + "group": "default", + "title": "uBlock filters – Privacy", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt", + "assets/ublock/privacy.txt" + ] }, - "CHN-2": { + "ublock-unbreak": { "content": "filters", - "off": true, - "title": "CHN: CJX's Annoyance List", - "group": "regions", - "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt", - "supportURL": "https://github.com/cjx82630/cjxlist" + "group": "default", + "title": "uBlock filters – Unbreak", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt", + "assets/ublock/unbreak.txt" + ] }, - "DEU-0": { + "awrl-0": { "content": "filters", + "group": "ads", "off": true, - "title": "DEU: EasyList Germany", - "group": "regions", - "lang": "de", - "contentURL": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", - "supportURL": "https://forums.lanik.us/viewforum.php?f=90" + "title": "Adblock Warning Removal List", + "contentURL": "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt", + "supportURL": "https://forums.lanik.us/" }, - "DNK-0": { + "reek-0": { "content": "filters", + "group": "ads", "off": true, - "title": "DNK: Schacks Adblock Plus liste", - "group": "regions", - "lang": "da", - "contentURL": "https://adblock.dk/block.csv", - "supportURL": "https://henrik.schack.dk/adblock/" + "title": "Anti-Adblock Killer | Reek", + "contentURL": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", + "supportURL": "https://github.com/reek/anti-adblock-killer", + "instructionURL": "https://github.com/reek/anti-adblock-killer#instruction" }, "easylist": { "content": "filters", - "title": "EasyList", "group": "ads", + "title": "EasyList", "contentURL": [ "https://easylist.to/easylist/easylist.txt", "https://easylist-downloads.adblockplus.org/easylist.txt", @@ -130,16 +104,23 @@ }, "easylist-nocosmetic": { "content": "filters", + "group": "ads", "off": true, "title": "EasyList without element hiding rules", - "group": "ads", "contentURL": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt", "supportURL": "https://forums.lanik.us/" }, + "disconnect-tracking": { + "content": "filters", + "group": "privacy", + "off": true, + "title": "Basic tracking list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" + }, "easyprivacy": { "content": "filters", - "title": "EasyPrivacy", "group": "privacy", + "title": "EasyPrivacy", "contentURL": [ "https://easylist.to/easylist/easyprivacy.txt", "https://easylist-downloads.adblockplus.org/easyprivacy.txt", @@ -148,427 +129,457 @@ ], "supportURL": "https://forums.lanik.us/" }, - "fanboy-annoyance": { + "fanboy-enhanced": { "content": "filters", + "group": "privacy", "off": true, - "title": "Fanboy’s Annoyance List", - "group": "social", - "contentURL": [ - "https://easylist.to/easylist/fanboy-annoyance.txt", - "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt" - ], + "title": "Fanboy’s Enhanced Tracking List", + "contentURL": "https://www.fanboy.co.nz/enhancedstats.txt", "supportURL": "https://forums.lanik.us/" }, - "fanboy-social": { + "disconnect-malvertising": { "content": "filters", + "group": "malware", "off": true, - "title": "Fanboy’s Social Blocking List", - "group": "social", + "title": "Malvertising filter list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt" + }, + "malware-0": { + "content": "filters", + "group": "malware", + "title": "Malware Domain List", "contentURL": [ - "https://easylist.to/easylist/fanboy-social.txt", - "https://easylist-downloads.adblockplus.org/fanboy-social.txt" - ], - "supportURL": "https://forums.lanik.us/" + "https://www.malwaredomainlist.com/hostslist/hosts.txt", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt" + ] }, - "FRA-0": { + "malware-1": { "content": "filters", - "off": true, - "title": "FRA: EasyList Liste FR", - "group": "regions", - "lang": "fr", - "contentURL": "https://easylist-downloads.adblockplus.org/liste_fr.txt", - "supportURL": "https://forums.lanik.us/viewforum.php?f=91" + "group": "malware", + "title": "Malware domains", + "contentURL": [ + "https://mirror.cedia.org.ec/malwaredomains/justdomains", + "https://mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt" + ], + "supportURL": "http://www.malwaredomains.com/" }, - "ISL-0": { + "malware-2": { "content": "filters", + "group": "malware", "off": true, - "title": "ISL: Icelandic ABP List", - "group": "regions", - "lang": "is", - "contentURL": "http://adblock.gardar.net/is.abp.txt", - "supportURL": "http://adblock.gardar.net/" + "title": "Malware domains (long-lived)", + "contentURL": [ + "https://mirror1.malwaredomains.com/files/immortal_domains.txt", + "https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt" + ], + "supportURL": "http://www.malwaredomains.com/" }, - "ITA-0": { + "disconnect-malware": { "content": "filters", + "group": "malware", "off": true, - "title": "ITA: EasyList Italy", - "group": "regions", - "lang": "it", - "contentURL": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", - "supportURL": "https://forums.lanik.us/viewforum.php?f=96" + "title": "Malware filter list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt" }, - "ITA-1": { + "spam404-0": { "content": "filters", + "group": "malware", "off": true, - "title": "ITA: ABP X Files", - "group": "regions", - "contentURL": "https://raw.githubusercontent.com/gioxx/xfiles/master/filtri.txt", - "supportURL": "http://noads.it/" + "title": "Spam404", + "contentURL": "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt", + "supportURL": "http://www.spam404.com/" }, - "RUS-0": { + "fanboy-thirdparty_social": { "content": "filters", + "group": "social", "off": true, - "title": "RUS: RU AdList (Дополнительная региональная подписка)", - "group": "regions", - "lang": "ru", - "contentURL": "https://easylist-downloads.adblockplus.org/advblock.txt", - "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + "title": "Anti-ThirdpartySocial (see warning inside list)", + "contentURL": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", + "supportURL": "https://forums.lanik.us/" }, - "RUS-1": { + "fanboy-annoyance": { "content": "filters", + "group": "social", "off": true, - "title": "RUS: BitBlock List (Дополнительная подписка фильтров)", - "group": "regions", - "contentURL": "https://easylist-downloads.adblockplus.org/bitblock.txt", - "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + "title": "Fanboy’s Annoyance List", + "contentURL": [ + "https://easylist.to/easylist/fanboy-annoyance.txt", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt" + ], + "supportURL": "https://forums.lanik.us/" }, - "RUS-2": { + "fanboy-social": { "content": "filters", + "group": "social", "off": true, - "title": "RUS: Adguard Russian Filter", - "group": "regions", - "contentURL": "https://filters.adtidy.org/extension/chromium/filters/1.txt", - "supportURL": "https://forum.adguard.com/forumdisplay.php?69-%D0%A4%D0%B8%D0%BB%D1%8C%D1%82%D1%80%D1%8B-Adguard" + "title": "Fanboy’s Social Blocking List", + "contentURL": [ + "https://easylist.to/easylist/fanboy-social.txt", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt" + ], + "supportURL": "https://forums.lanik.us/" }, - "NLD-0": { + "dpollock-0": { "content": "filters", + "group": "multipurpose", + "updateAfter": 11, "off": true, - "title": "NLD: EasyList Dutch", - "group": "regions", - "lang": "nl", - "contentURL": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", - "supportURL": "https://forums.lanik.us/viewforum.php?f=100" + "title": "Dan Pollock’s hosts file", + "contentURL": "http://someonewhocares.org/hosts/hosts", + "supportURL": "http://someonewhocares.org/hosts/" }, - "LVA-0": { + "fanboy-ultimate": { "content": "filters", + "group": "multipurpose", "off": true, - "title": "LVA: Latvian List", - "group": "regions", - "lang": "lv", - "contentURL": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", - "supportURL": "https://notabug.org/latvian-list/adblock-latvian" + "title": "Fanboy+Easylist-Merged Ultimate List", + "contentURL": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", + "supportURL": "https://forums.lanik.us/" }, "hphosts": { "content": "filters", + "group": "multipurpose", + "updateAfter": 11, "off": true, "title": "hpHosts’ Ad and tracking servers", - "group": "multipurpose", "contentURL": "https://hosts-file.net/.%5Cad_servers.txt", "supportURL": "https://hosts-file.net/" }, - "EST-0": { - "content": "filters", - "off": true, - "title": "EST: Eesti saitidele kohandatud filter", - "group": "regions", - "lang": "et", - "contentURL": "http://adblock.ee/list.php", - "supportURL": "http://adblock.ee/" - }, - "disconnect-malvertising": { + "mvps-0": { "content": "filters", + "group": "multipurpose", + "updateAfter": 11, "off": true, - "title": "Malvertising filter list by Disconnect", - "group": "malware", - "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt" + "title": "MVPS HOSTS", + "contentURL": "http://winhelp2002.mvps.org/hosts.txt", + "supportURL": "http://winhelp2002.mvps.org/" }, - "disconnect-malware": { + "plowe-0": { "content": "filters", - "off": true, - "title": "Malware filter list by Disconnect", - "group": "malware", - "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt" + "group": "multipurpose", + "updateAfter": 13, + "title": "Peter Lowe’s Ad and tracking server list", + "contentURL": [ + "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext", + "assets/thirdparties/pgl.yoyo.org/as/serverlist", + "assets/thirdparties/pgl.yoyo.org/as/serverlist.txt" + ], + "supportURL": "https://pgl.yoyo.org/adservers/" }, - "disconnect-tracking": { + "ara-0": { "content": "filters", + "group": "regions", "off": true, - "title": "Basic tracking list by Disconnect", - "group": "privacy", - "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" + "title": "ara: Liste AR", + "lang": "ar", + "contentURL": "https://easylist-downloads.adblockplus.org/Liste_AR.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=98" }, - "POL-0": { + "BGR-0": { "content": "filters", - "off": true, - "title": "POL: polskie filtry do Adblocka i uBlocka", "group": "regions", - "lang": "pl", - "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", - "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/" - }, - "awrl-0": { - "content": "filters", "off": true, - "title": "Adblock Warning Removal List", - "group": "ads", - "contentURL": "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt", - "supportURL": "https://forums.lanik.us/" + "title": "BGR: Bulgarian Adblock list", + "lang": "bg", + "contentURL": "https://stanev.org/abp/adblock_bg.txt", + "supportURL": "https://stanev.org/abp/" }, - "FIN-0": { + "CHN-0": { "content": "filters", - "off": true, - "title": "FIN: Finnish Addition to Easylist", "group": "regions", - "lang": "fi", - "contentURL": "http://adb.juvander.net/Finland_adb.txt", - "supportURL": "http://www.juvander.fi/AdblockFinland" + "off": true, + "title": "CHN: EasyList China (中文)", + "lang": "zh", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistchina.txt", + "supportURL": "http://abpchina.org/forum/forum.php" }, - "KOR-0": { + "CHN-1": { "content": "filters", - "off": true, - "title": "KOR: Korean Adblock List", "group": "regions", - "lang": "ko", - "contentURL": "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt", - "supportURL": "https://github.com/gfmaster/adblock-korea-contrib" + "off": true, + "title": "CHN: CJX's EasyList Lite (main focus on Chinese sites)", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" }, - "KOR-1": { + "CHN-2": { "content": "filters", - "off": true, - "title": "KOR: YousList", "group": "regions", - "lang": "ko", - "contentURL": "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt", - "supportURL": "https://github.com/yous/YousList" + "off": true, + "title": "CHN: CJX's Annoyance List", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" }, - "KOR-2": { + "CZE-0": { "content": "filters", - "off": true, - "title": "KOR: Fanboy's Korean", "group": "regions", - "contentURL": "https://www.fanboy.co.nz/fanboy-korean.txt", - "supportURL": "https://forums.lanik.us/" + "off": true, + "title": "CZE, SVK: EasyList Czech and Slovak", + "lang": "cs", + "contentURL": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", + "supportURL": "https://github.com/tomasko126/easylistczechandslovak" }, - "IDN-0": { + "DEU-0": { "content": "filters", - "off": true, - "title": "IDN: ABPindo", "group": "regions", - "lang": "id", + "off": true, + "title": "DEU: EasyList Germany", + "lang": "de", "contentURL": [ - "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt", - "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt" + "https://easylist.to/easylistgermany/easylistgermany.txt", + "https://easylist-downloads.adblockplus.org/easylistgermany.txt" ], - "supportURL": "https://github.com/ABPindo/indonesianadblockrules" + "supportURL": "https://forums.lanik.us/viewforum.php?f=90" }, - "JPN-0": { + "DNK-0": { "content": "filters", + "group": "regions", "off": true, - "title": "JPN: ABP Japanese filters (日本用フィルタ)", + "title": "DNK: Schacks Adblock Plus liste", + "lang": "da", + "contentURL": "https://adblock.dk/block.csv", + "supportURL": "https://henrik.schack.dk/adblock/" + }, + "EST-0": { + "content": "filters", "group": "regions", - "lang": "ja", - "contentURL": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", - "supportURL": "https://github.com/k2jp/abp-japanese-filters/wiki/Support_Policy" + "off": true, + "title": "EST: Eesti saitidele kohandatud filter", + "lang": "et", + "contentURL": "http://adblock.ee/list.php", + "supportURL": "http://adblock.ee/" }, "EU-prebake": { "content": "filters", + "group": "regions", "off": true, "title": "EU: Prebake - Filter Obtrusive Cookie Notices", - "group": "regions", "contentURL": "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt", "supportURL": "https://github.com/liamja/Prebake" }, - "ara-0": { + "FIN-0": { "content": "filters", - "off": true, - "title": "ara: Liste AR", "group": "regions", - "lang": "ar", - "contentURL": "https://easylist-downloads.adblockplus.org/Liste_AR.txt", - "supportURL": "https://forums.lanik.us/viewforum.php?f=98" + "off": true, + "title": "FIN: Finnish Addition to Easylist", + "lang": "fi", + "contentURL": "http://adb.juvander.net/Finland_adb.txt", + "supportURL": "http://www.juvander.fi/AdblockFinland" }, - "LTU-0": { + "FRA-0": { "content": "filters", - "off": true, - "title": "LTU: Adblock Plus Lithuania", "group": "regions", - "lang": "lt", - "contentURL": "http://margevicius.lt/easylistlithuania.txt", - "supportURL": "http://margevicius.lt/easylist_lithuania/" + "off": true, + "title": "FRA: EasyList Liste FR", + "lang": "fr", + "contentURL": "https://easylist-downloads.adblockplus.org/liste_fr.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=91" }, - "malware-0": { + "GRC-0": { "content": "filters", + "group": "regions", "off": true, - "title": "Malware domains (long-lived)", - "group": "malware", - "contentURL": "http://malwaredomains.lehigh.edu/files/immortal_domains.txt", - "supportURL": "http://www.malwaredomains.com/" + "title": "GRC: Greek AdBlock Filter", + "lang": "el", + "contentURL": "https://www.void.gr/kargig/void-gr-filters.txt", + "supportURL": "https://github.com/kargig/greek-adblockplus-filter" }, - "malware-1": { + "HUN-0": { "content": "filters", - "title": "Malware Domain List", - "group": "malware", - "contentURL": [ - "https://www.malwaredomainlist.com/hostslist/hosts.txt", - "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt" - ] + "group": "regions", + "off": true, + "title": "HUN: hufilter", + "lang": "hu", + "contentURL": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", + "supportURL": "https://github.com/szpeter80/hufilter" }, - "malware-2": { + "IDN-0": { "content": "filters", - "title": "Malware domains", - "group": "malware", + "group": "regions", + "off": true, + "title": "IDN: ABPindo", + "lang": "id", "contentURL": [ - "https://mirror.cedia.org.ec/malwaredomains/justdomains", - "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains", - "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt" + "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt" ], - "supportURL": "http://www.malwaredomains.com/" + "supportURL": "https://github.com/ABPindo/indonesianadblockrules" }, - "plowe-0": { + "ISL-0": { "content": "filters", - "title": "Peter Lowe’s Ad and tracking server list", - "group": "multipurpose", - "contentURL": [ - "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext", - "assets/thirdparties/pgl.yoyo.org/as/serverlist", - "assets/thirdparties/pgl.yoyo.org/as/serverlist.txt" - ], - "supportURL": "https://pgl.yoyo.org/adservers/" + "group": "regions", + "off": true, + "title": "ISL: Icelandic ABP List", + "lang": "is", + "contentURL": "http://adblock.gardar.net/is.abp.txt", + "supportURL": "http://adblock.gardar.net/" }, "ISR-0": { "content": "filters", + "group": "regions", "off": true, "title": "ISR: EasyList Hebrew", - "group": "regions", "lang": "he", "contentURL": "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt", "supportURL": "https://github.com/easylist/EasyListHebrew" }, - "reek-0": { + "ITA-0": { "content": "filters", + "group": "regions", "off": true, - "title": "Anti-Adblock Killer | Reek", - "group": "ads", - "contentURL": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", - "supportURL": "https://github.com/reek/anti-adblock-killer", - "instructionURL": "https://github.com/reek/anti-adblock-killer#instruction" + "title": "ITA: EasyList Italy", + "lang": "it", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=96" }, - "HUN-0": { + "ITA-1": { "content": "filters", + "group": "regions", "off": true, - "title": "HUN: hufilter", + "title": "ITA: ABP X Files", + "contentURL": "https://raw.githubusercontent.com/gioxx/xfiles/master/filtri.txt", + "supportURL": "http://noads.it/" + }, + "JPN-0": { + "content": "filters", "group": "regions", - "lang": "hu", - "contentURL": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", - "supportURL": "https://github.com/szpeter80/hufilter" + "off": true, + "title": "JPN: ABP Japanese filters (日本用フィルタ)", + "lang": "ja", + "contentURL": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", + "supportURL": "https://github.com/k2jp/abp-japanese-filters/wiki/Support_Policy" }, - "CZE-0": { + "KOR-0": { "content": "filters", + "group": "regions", "off": true, - "title": "CZE, SVK: EasyList Czech and Slovak", + "title": "KOR: Korean Adblock List", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt", + "supportURL": "https://github.com/gfmaster/adblock-korea-contrib" + }, + "KOR-1": { + "content": "filters", "group": "regions", - "lang": "cs", - "contentURL": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", - "supportURL": "https://github.com/tomasko126/easylistczechandslovak" + "off": true, + "title": "KOR: YousList", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt", + "supportURL": "https://github.com/yous/YousList" }, - "dpollock-0": { + "KOR-2": { "content": "filters", + "group": "regions", "off": true, - "title": "Dan Pollock’s hosts file", - "group": "multipurpose", - "contentURL": "http://someonewhocares.org/hosts/hosts", - "supportURL": "http://someonewhocares.org/hosts/" + "title": "KOR: Fanboy's Korean", + "contentURL": "https://www.fanboy.co.nz/fanboy-korean.txt", + "supportURL": "https://forums.lanik.us/" }, - "spam404-0": { + "LTU-0": { "content": "filters", + "group": "regions", "off": true, - "title": "Spam404", - "group": "malware", - "contentURL": "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt", - "supportURL": "http://www.spam404.com/" + "title": "LTU: Adblock Plus Lithuania", + "lang": "lt", + "contentURL": "http://margevicius.lt/easylistlithuania.txt", + "supportURL": "http://margevicius.lt/easylist_lithuania/" }, - "BGR-0": { + "LVA-0": { "content": "filters", + "group": "regions", "off": true, - "title": "BGR: Bulgarian Adblock list", + "title": "LVA: Latvian List", + "lang": "lv", + "contentURL": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", + "supportURL": "https://notabug.org/latvian-list/adblock-latvian" + }, + "NLD-0": { + "content": "filters", "group": "regions", - "lang": "bg", - "contentURL": "https://stanev.org/abp/adblock_bg.txt", - "supportURL": "https://stanev.org/abp/" + "off": true, + "title": "NLD: EasyList Dutch", + "lang": "nl", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=100" }, - "mvps-0": { + "POL-0": { "content": "filters", + "group": "regions", "off": true, - "title": "MVPS HOSTS", - "group": "multipurpose", - "contentURL": "http://winhelp2002.mvps.org/hosts.txt", - "supportURL": "http://winhelp2002.mvps.org/" + "title": "POL: polskie filtry do Adblocka i uBlocka", + "lang": "pl", + "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", + "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/" }, - "fanboy-enhanced": { + "RUS-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "RUS: RU AdList (Дополнительная региональная подписка)", + "lang": "ru", + "contentURL": "https://easylist-downloads.adblockplus.org/advblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + }, + "RUS-1": { "content": "filters", + "group": "regions", "off": true, - "title": "Fanboy’s Enhanced Tracking List", - "group": "privacy", - "contentURL": "https://www.fanboy.co.nz/enhancedstats.txt", - "supportURL": "https://forums.lanik.us/" + "title": "RUS: BitBlock List (Дополнительная подписка фильтров)", + "contentURL": "https://easylist-downloads.adblockplus.org/bitblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" }, - "fanboy-thirdparty_social": { + "RUS-2": { "content": "filters", + "group": "regions", "off": true, - "title": "Anti-ThirdpartySocial (see warning inside list)", - "group": "social", - "contentURL": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", - "supportURL": "https://forums.lanik.us/" + "title": "RUS: Adguard Russian Filter", + "contentURL": "https://filters.adtidy.org/extension/chromium/filters/1.txt", + "supportURL": "https://forum.adguard.com/forumdisplay.php?69-%D0%A4%D0%B8%D0%BB%D1%8C%D1%82%D1%80%D1%8B-Adguard" }, "spa-0": { "content": "filters", + "group": "regions", "off": true, "title": "spa: EasyList Spanish", - "group": "regions", "lang": "es", "contentURL": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", "supportURL": "https://forums.lanik.us/viewforum.php?f=103" }, + "SVN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "SVN: Slovenian List", + "lang": "sl", + "contentURL": "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt", + "supportURL": "https://github.com/betterwebleon/slovenian-list" + }, "SWE-0": { "content": "filters", + "group": "regions", "off": true, "title": "SWE: Fanboy's Swedish", - "group": "regions", "lang": "sv", "contentURL": "https://www.fanboy.co.nz/fanboy-swedish.txt", "supportURL": "https://forums.lanik.us/" }, - "fanboy-ultimate": { - "content": "filters", - "off": true, - "title": "Fanboy+Easylist-Merged Ultimate List", - "group": "multipurpose", - "contentURL": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", - "supportURL": "https://forums.lanik.us/" - }, "TUR-0": { "content": "filters", + "group": "regions", "off": true, "title": "TUR: Adguard Turkish Filter", - "group": "regions", "lang": "tr", "contentURL": "https://filters.adtidy.org/extension/chromium/filters/13.txt", "supportURL": "https://forum.adguard.com/forumdisplay.php?51-Filter-Rules" }, "VIE-0": { "content": "filters", + "group": "regions", "off": true, "title": "VIE: Fanboy's Vietnamese", - "group": "regions", "lang": "vi", "contentURL": "https://www.fanboy.co.nz/fanboy-vietnam.txt", "supportURL": "https://forums.lanik.us/" - }, - "GRC-0": { - "content": "filters", - "off": true, - "title": "GRC: Greek AdBlock Filter", - "group": "regions", - "lang": "el", - "contentURL": "https://www.void.gr/kargig/void-gr-filters.txt", - "supportURL": "https://github.com/kargig/greek-adblockplus-filter" - }, - "SVN-0": { - "content": "filters", - "off": true, - "title": "SVN: Slovenian List", - "group": "regions", - "lang": "sl", - "contentURL": "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt", - "supportURL": "https://github.com/betterwebleon/slovenian-list" } } From c8317f17bcee2be5b56dca671034db02f570d157 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 12:06:05 -0500 Subject: [PATCH 14/15] ability to customize updater through advanced settings --- src/js/3p-filters.js | 11 +++++------ src/js/assets.js | 22 ++++++++++------------ src/js/background.js | 4 ++++ src/js/messaging.js | 2 +- src/js/storage.js | 17 +++++++++++++++-- 5 files changed, 35 insertions(+), 21 deletions(-) diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 5f307f14e0fa0..4b7dfa1b9dfea 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -166,7 +166,9 @@ var renderFilterLists = function() { liGroup.querySelector('.geName').textContent = groupName; } } - liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys); + if ( liGroup.querySelector('.geName:empty') === null ) { + liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys); + } var ulGroup = liGroup.querySelector('.listEntries'); if ( !listKeys ) { return liGroup; } listKeys.sort(function(a, b) { @@ -390,13 +392,10 @@ var buttonApplyHandler = function() { /******************************************************************************/ -var buttonUpdateHandler = function(ev) { +var buttonUpdateHandler = function() { var onSelectionDone = function() { uDom('#lists .listEntry.obsolete').addClass('updating'); - messaging.send('dashboard', { - what: 'forceUpdateAssets', - fast: ev.ctrlKey - }); + messaging.send('dashboard', { what: 'forceUpdateAssets' }); }; selectFilterLists(onSelectionDone); }; diff --git a/src/js/assets.js b/src/js/assets.js index fb86926ac51a3..f4daf8f28f7dc 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -27,10 +27,9 @@ /******************************************************************************/ -var reIsExternalPath = /^(?:file|ftps?|https?|resource):\/\//; -var reIsUserAsset = /^user-/; -var errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); -var xhrTimeout = vAPI.localStorage.getItem('xhrTimeout') || 30000; +var reIsExternalPath = /^(?:[a-z-]+):\/\//, + reIsUserAsset = /^user-/, + errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); var api = { }; @@ -107,7 +106,7 @@ var getTextFileFromURL = function(url, onLoad, onError) { var xhr = new XMLHttpRequest(); try { xhr.open('get', url, true); - xhr.timeout = xhrTimeout; + xhr.timeout = µBlock.hiddenSettings.assetFetchTimeout * 1000 || 30000; xhr.onload = onResponseReceived; xhr.onerror = onErrorReceived; xhr.ontimeout = onErrorReceived; @@ -890,13 +889,12 @@ api.rmrf = function() { /******************************************************************************/ // Asset updater area. - -var updaterStatus; -var updaterTimer; -var updaterAssetDelayDefault = 2 * 60 * 1000; -var updaterAssetDelay = updaterAssetDelayDefault; -var updaterUpdated = []; -var updaterFetched = new Set(); +var updaterStatus, + updaterTimer, + updaterAssetDelayDefault = 120000, + updaterAssetDelay = updaterAssetDelayDefault, + updaterUpdated = [], + updaterFetched = new Set(); var updateFirst = function() { updaterStatus = 'updating'; diff --git a/src/js/background.js b/src/js/background.js index c82fa051e05db..791e56be94d6f 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -67,8 +67,12 @@ return { }, hiddenSettingsDefault: { + assetFetchTimeout: 30, + autoUpdateAssetFetchPeriod: 120, + autoUpdatePeriod: 7, ignoreRedirectFilters: false, ignoreScriptInjectFilters: false, + manualUpdateAssetFetchPeriod: 2000, popupFontSize: 'unset', suspendTabsUntilReady: false }, diff --git a/src/js/messaging.js b/src/js/messaging.js index 6ab60da30976e..4cc466f7c4314 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -122,7 +122,7 @@ var onMessage = function(request, sender, callback) { case 'forceUpdateAssets': µb.scheduleAssetUpdater(0); - µb.assets.updateStart({ delay: request.fast ? 1 : 2000 }); + µb.assets.updateStart({ delay: µb.hiddenSettings.manualUpdateAssetFetchPeriod || 2000 }); break; case 'getAppData': diff --git a/src/js/storage.js b/src/js/storage.js index 43421bcf8dff6..c587919487353 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -104,6 +104,12 @@ case 'string': out[name] = value; break; + case 'number': + out[name] = parseInt(value, 10); + if ( isNaN(out[name]) ) { + out[name] = this.hiddenSettingsDefault[name]; + } + break; default: break; } @@ -977,7 +983,10 @@ timer = vAPI.setTimeout(function() { timer = undefined; next = 0; - µBlock.assets.updateStart({ delay: 2 * 60 * 1000 }); + var µb = µBlock; + µb.assets.updateStart({ + delay: µb.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || 120000 + }); }, updateDelay); }; })(); @@ -1037,7 +1046,11 @@ if ( details.assetKeys.length !== 0 ) { this.loadFilterLists(); } - this.scheduleAssetUpdater(this.userSettings.autoUpdate ? 11 * 60 * 60 * 1000 : 0); + if ( this.userSettings.autoUpdate ) { + this.scheduleAssetUpdater(this.hiddenSettings.assetAutoUpdatePeriod * 3600000 || 25200000); + } else { + this.scheduleAssetUpdater(0); + } return; } }; From 65be357f8ad5ebdc943ff2365304169909c08737 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 12:24:26 -0500 Subject: [PATCH 15/15] better spinner icon --- src/3p-filters.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3p-filters.html b/src/3p-filters.html index 2cfd6b058d054..48086f71fe271 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -54,7 +54,7 @@  -->http +  -->