README.md

@@ -4,6 +4,12 @@ A browser-based solution to Web censorship, implemented as a JavaScript library
 
 Ideally, users should not need to install any special software nor change any settings to continue being able to access a blocked Samizdat-enabled site as soon as they are able to access it *once*.
 
+## Curent status
+
+Samizdat is currently considered *alpha*: the code works, but major rewrites and API changes are coming.
+
+Feel free to test it, but be aware that it might not work as expected. If you'd like to get in touch, please e-mail us at `rysiek+samizdat[at]occrp.org`.
+
 ## Rationale
 
 While a number of censorship circumvention technologies exist, these typically require those who want to access the blocked content (readers) to install specific tools (applications, browser extensions, VPN software, etc.), or change their settings (DNS servers, HTTP proxies, etc.). This approach does not scale.

css/style.css

@@ -19,6 +19,14 @@ h1 {
     text-align:center;
 }
 
+code {
+    background: #fff6;
+    padding: 0.2em 0.5em;
+    border-radius: 0.4em;
+    text-shadow: none;
+    font-weight: bold;
+}
+
 #subtitle {
     font-style: italic;
     font-variant: super;

index.html

@@ -15,6 +15,7 @@
   <script defer src="./lib/gun.js"></script>
   <script defer src="./lib/sea.js"></script>
   <script defer src="./lib/webrtc.js"></script>
+  <script defer src="./plugins/fetch.js"></script>
   <script defer src="./plugins/cache.js"></script>
   <script defer src="./plugins/gun-ipfs.js"></script>
   <script>
@@ -66,7 +67,7 @@
         if (typeof fetchedResourcesDisplay !== 'object') {
           fetchedResourcesDisplay = document.getElementById("fetched-resources-list")
         }
-        var itemHTML = `<li class="fetched-resources-item"><label><input type="checkbox" checked="checked"/><span class="fetched-resource-url"><span>${si.url}</span></span><span class="fetched-resource-method fetch ${(si.method === 'fetch') ? 'active' : ''}">fetch</span>`
+        var itemHTML = `<li class="fetched-resources-item"><label><input type="checkbox" checked="checked"/><span class="fetched-resource-url"><span>${si.url}</span></span>`
         SamizdatPlugins.forEach((plugin)=>{
           var pclass = samizdat.safeClassName(plugin.name);
           itemHTML += `<span class="fetched-resource-method ${pclass} ${(si.method === plugin.name) ? 'active' : ''}">${plugin.name}</span>`
@@ -183,33 +184,35 @@
       
 
       /**
-       * adding certain resources from cache
+       * stashing and unstashing resources
+       * 
+       * stash param means "stash" if set to true (the default), "unstash" otherwise
        */
-      samizdat.addResourcesToCache = () => {
-        caches.open('v1')
-            .then((cache) => {
-                var resources = []
-                document.querySelectorAll('.fetched-resources-item input:checked')
-                    .forEach((el)=>{
-                        resources.push(el.parentElement.querySelector('.fetched-resource-url').innerText)
-                    })
-                return SamizdatPlugins[0].push(resources)
-            })
-      }
-      
-      /**
-       * removing certain resources from cache
-       */
-      samizdat.clearResourcesFromCache = () => {
-        caches.open('v1')
-            .then((cache) => {
-                document.querySelectorAll('.fetched-resources-item input:checked')
-                    .forEach((el)=>{
-                        cache.delete(el.parentElement.querySelector('.fetched-resource-url').innerText)
-                    })
-            })
+      samizdat.stashOrUnstashResources = (stash=true) => {
+          // what are we doing?
+          if (stash) {
+              var operation = 'stash'
+          } else {
+              var operation = 'unstash'
+          }
+          // get the resources
+          var resources = []
+          document
+              .querySelectorAll('.fetched-resources-item input:checked')
+              .forEach((el)=>{
+                  resources.push(el.parentElement.querySelector('.fetched-resource-url').innerText)
+              })
+          // cycle through plugins and find the first that implements a stash() method
+          for (i=0; i<SamizdatPlugins.length; i++) {
+              if (typeof SamizdatPlugins[i][operation] === 'function') {
+                  console.log('(COMMIT_UNKNOWN) Using plugin "' + SamizdatPlugins[i].name + '" to ' + operation + ' the resources...')
+                  return SamizdatPlugins[i][operation](resources)
+              } 
+          }
+          // if we're here that means there was no plugin able to stash things
+          return Promise.reject(new Error('No stashing plugin found'))
       }
-      
+
       
       /**
        * publishing certain resources to Gun+IPFS
@@ -225,15 +228,11 @@
             .forEach((el)=>{
                 resources.push(el.parentElement.querySelector('.fetched-resource-url').innerText)
             })
-        return SamizdatPlugins[1].push(resources, user, pass)
+        return SamizdatPlugins[1].publish(resources, user, pass)
       }
       
 
       // add plugin status display
-      samizdat.addPluginStatus({
-        name: 'fetch',
-        description: 'Regular HTTP(S) fetch()'
-      })
       SamizdatPlugins.forEach(samizdat.addPluginStatus)
 
       // TODO: do it better, watch for ongoing requests or some such?
@@ -260,12 +259,7 @@
         )
         // once we have all the data...
           .then((val)=>{
-            // add plugin stats for the regular HTTP(S) fetch()
-            samizdat.updatePluginStatus({
-              name: 'fetch',
-              description: 'Regular HTTP(S) fetch()'
-            })
-            // and all other plugins
+            // add plugins' stats
             SamizdatPlugins.forEach(samizdat.updatePluginStatus)
           })
       }, 5000)
@@ -283,6 +277,7 @@
     <p><em>Samizdat</em> is a browser-based Web censorship circumvention library, easily deployable on any website.</p>
     <p>Implemented in JavaScript, It uses  <a href="https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API/Using_Service_Workers">Service Workers</a> and a set of non-standard in-browser content delivery mechanisms (with strong focus on decentralized ones, like <a href="https://gun.eco/">Gun</a>, and <a href="https://github.com/ipfs/js-ipfs">JS-IPFS</a>).</p>
     <p>Ideally, users would not need to install any special software nor change any settings to continue being able to access a blocked <em>Samizdat</em>-enabled site as soon as they are able to access it <em>once</em>.</p>
+    <p><em>Samizdat</em> is currently considered <code>alpha</code> software. We would love to hear if you'd like to test it &mdash; you can contact us on <code>rysiek+samizdat[at]occrp.org</code>.<p>
   </div>
   <div id="logo-container">
     <div id="logo">
@@ -299,7 +294,7 @@
     <p>The list below contains all resources fetched in relation to this page.</p>
     <ul id="fetched-resources-list"></ul>
     <p id="fetched-resources-list-empty">The list is empty, but if the Service Worker is running it should be populated soon.</p>
-    <p id="fetched-resources-controls"><button type="button" onclick="samizdat.toggleResourceCheckboxes()">Toggle selection</button><span class="spacer"></span><button type="button" onclick="samizdat.addResourcesToCache()">Add selected to cache</button><button type="button" onclick="samizdat.clearResourcesFromCache()">Clear selected from cache</button><span class="spacer"></span><input type="text" placeholder="Gun username" id="samizdat-gun-user"/><input type="password" placeholder="Gun password" id="samizdat-gun-password"/><button type="button" onclick="samizdat.publishResourcesToGunAndIPFS()">Publish to Gun+IPFS</button></p>
+    <p id="fetched-resources-controls"><button type="button" onclick="samizdat.toggleResourceCheckboxes()">Toggle selection</button><span class="spacer"></span><button type="button" onclick="samizdat.stashOrUnstashResources(true)">Add selected to cache</button><button type="button" onclick="samizdat.stashOrUnstashResources(false)">Clear selected from cache</button><span class="spacer"></span><input type="text" placeholder="Gun username" id="samizdat-gun-user"/><input type="password" placeholder="Gun password" id="samizdat-gun-password"/><button type="button" onclick="samizdat.publishResourcesToGunAndIPFS()">Publish to Gun+IPFS</button></p>
   </div>
   <p id="footer">ServiceWorker: <span id="samizdat-commit-service-worker">NO_INFO</span>&nbsp;::&nbsp;index.html: <span id="samizdat-commit-index-html">COMMIT_UNKNOWN</span><br/>code: <span id="samizdat-code"><a href="https://git.occrp.org/libre/samizdat/">here</a></span>&nbsp;::&nbsp;license: <span id="samizdat-license"><a href="https://git.occrp.org/libre/samizdat/blob/master/LICENSE">AGPL</a></span></p>
 </body>

plugins/cache.js

+/* ========================================================================= *\
+|* === Stashing plugin using the Cache API                               === *|
+\* ========================================================================= */
+
 /**
  * getting content from cache
  */
@@ -17,9 +21,13 @@ let getContentFromCache = (url) => {
 }
 
 /**
+ * add resources to cache
+ * 
+ * implements the stash() Samizdat plugin method
+ * 
  * accepts either a Response
  * or a string containing a URL
- * or an array of string URLs
+ * or an Array of string URLs
  */
 let cacheContent = (resource, url) => {
     return caches.open('v1')
@@ -46,6 +54,40 @@ let cacheContent = (resource, url) => {
             }
         })
 }
+
+/**
+ * remove resources from cache
+ * 
+ * implements the unstash() Samizdat plugin method 
+ * 
+ * accepts either a Response
+ * or a string containing a URL
+ * or an Array of string URLs
+ */
+let clearCachedContent = (resource) => {
+    return caches.open('v1')
+        .then((cache) => {
+            if (typeof resource === 'string') {
+                // assume URL
+                console.log("(COMMIT_UNKNOWN) deleting a cached URL")
+                return cache.delete(resource)
+            } else if (Array.isArray(resource)) {
+                // assume array of URLs
+                console.log("(COMMIT_UNKNOWN) deleting an Array of cached URLs")
+                return Promise.all(
+                    resource.map((res)=>{
+                        return cache.delete(res)
+                    })
+                )
+            } else {
+                // assume a Response
+                // which means we have an URL in resource.url
+                console.log("(COMMIT_UNKNOWN) removing a Response from cache: " + resource.url)
+                return cache.delete(resource.url)
+            }
+        })
+}
+
  
 // initialize the SamizdatPlugins array
 if (!Array.isArray(self.SamizdatPlugins)) {
@@ -59,5 +101,6 @@ self.SamizdatPlugins.push({
     description: 'Locally cached responses, using the Cache API.',
     version: 'COMMIT_UNKNOWN',
     fetch: getContentFromCache,
-    push: cacheContent
+    stash: cacheContent,
+    unstash: clearCachedContent
 })

plugins/fetch.js

+/* ========================================================================= *\
+|* === Regular HTTP(S) fetch() plugin                                    === *|
+\* ========================================================================= */
+
+/**
+ * this plugin does not implement any push method
+ */
+
+/**
+ * getting content using regular HTTP(S) fetch()
+ */
+let fetchContent = (url) => {
+    console.log('Samizdat: regular fetch!')
+    return fetch(url)
+        .then((response) => {
+            // 4xx? 5xx? that's a paddlin'
+            if (response.status >= 400) {
+                // throw an Error to fall back to Samizdat:
+                throw new Error('HTTP Error: ' + response.status + ' ' + response.statusText);
+            }
+            // all good, it seems
+            console.log("(COMMIT_UNKNOWN) Fetched:", response.url);
+            return response;
+        })
+}
+ 
+// initialize the SamizdatPlugins array
+if (!Array.isArray(self.SamizdatPlugins)) {
+    self.SamizdatPlugins = new Array()
+}
+
+// and add ourselves to it
+// with some additional metadata
+self.SamizdatPlugins.push({
+    name: 'fetch',
+    description: 'Just a regular HTTP(S) fetch()',
+    version: 'COMMIT_UNKNOWN',
+    fetch: fetchContent,
+})

plugins/gun-ipfs.js

@@ -421,5 +421,5 @@ self.SamizdatPlugins.push({
     description: 'Decentralized resource fetching using Gun for address resolution and IPFS for content delivery.',
     version: 'COMMIT_UNKNOWN',
     fetch: getContentFromGunAndIPFS,
-    push: publishContent
+    publish: publishContent
 })

service-worker.js

 /*
  * Samizdat Service Worker.
  *
- * Strategy:
+ * Strategy (not fully implemented yet):
  *    1. Try to load from main website.
  *    2. If loading fails, load from Samizdat.
  *    3. If loading is too slow, load from Samizdat.
@@ -18,12 +18,17 @@ if (!Array.isArray(self.SamizdatPlugins)) {
 // order in which plugins are loaded defines the order
 // in which they are called!
 self.importScripts(
+        "./plugins/fetch.js",
         "./plugins/cache.js",
         "./plugins/gun-ipfs.js",
         "./lib/idb-keyval-iife.min.js");
 
 console.log('(COMMIT_UNKNOWN) SamizdatPlugins.length:', self.SamizdatPlugins.length)
 
+/* ========================================================================= *\
+|* === SamizdatInfo                                                      === *|
+\* ========================================================================= */
+
 // initialize the Samizdat info key-value store
 if (typeof self.samizdatStore === 'undefined') {
     self.samizdatStore = new idbKeyval.Store('samizdat', 'info')
@@ -149,6 +154,158 @@ if (typeof SamizdatInfo !== "object") {
 }
 
 
+/* ========================================================================= *\
+|* === Main Brain of Samizdat                                            === *|
+\* ========================================================================= */
+
+/**
+ * get a plugin by name
+ * 
+ * this doesn't have to be super-performant, since we should never have more
+ * then a few plugins
+ * (let's see how long it takes for me to eat my own words here)
+ */
+let getSamizdatPluginByName = (name) => {
+    for (i=0; i<SamizdatPlugins.length; i++) {
+        if (SamizdatPlugins[i].name === name) {
+            return SamizdatPlugins[i]
+        }
+    }
+    return null
+}
+
+/**
+ * run a plugin's fetch() method
+ * while handling all the auxiliary stuff like saving info in SamizdatInfo
+ * 
+ * plugin     - the plugin to use
+ * url        - string containing the URL to fetch
+ * lastError  - error thrown by the previous plugin, if any (default: null)
+ * useStashed - use stashed resources; if false, error out on any plugin that implements stash() (default: true)
+ */
+let samizdatFetch = (plugin, url, lastError=null, useStashed=true) => {
+    // save info in SamizdatInfo
+    SamizdatInfo.resources[url].fetchError = lastError;
+    SamizdatInfo.resources[url].method = plugin.name
+    // log stuff
+    console.log("(COMMIT_UNKNOWN) Samizdat handling URL:", url);
+    console.log('+-- last error     : ' + lastError)
+    console.log('+-- current method : ' + plugin.name)
+    // do we want to use stashed resources?
+    // TODO: there's probably a better way of handling that than throwing an Error()
+    if (typeof plugin.stash === 'function' && ! useStashed) {
+        throw new Error('Not supposed to use stashed resources.')
+    }
+    // run the plugin
+    return plugin.fetch(url)
+}
+
+/**
+ * Cycles through all the plugins, in the order they got registered,
+ * and returns a Promise resolving to a Response in case any of the plugins
+ * was able to get the resource
+ * 
+ * url        - string containing the URL we want to fetch
+ * useStashed - use stashed resources; if false, only pull resources from live sources
+ * doStash    - stash resources once fetched successfully; if false, do not stash pulled resources automagically
+ */
+let getResourceThroughSamizdat = (url, useStashed=true, doStash=true) => {
+
+    // make sure SamizdatInfo exists for the url
+    if (typeof SamizdatInfo.resources[url] === 'undefined') {
+        SamizdatInfo.resources[url] = new SamizdatResourceInfo(url)
+        console.log('+-- created SamizdatInfo.resources[' + url + ']')
+    }
+    
+    /**
+     * this uses Array.reduce() to chain the SamizdatPlugins[]-generated Promises
+     * using the Promise the first registered plugin as the default value
+     * 
+     * see: https://css-tricks.com/why-using-reduce-to-sequentially-resolve-promises-works/
+     * 
+     * this also means that SamizdatPlugins[0].fetch() below will run first
+     * (counter-intutively!)
+     */
+    return SamizdatPlugins.reduce(
+        (prevPromise, currentPlugin)=>{
+            return prevPromise.catch((error)=>{
+                return samizdatFetch(currentPlugin, url, error.toString(), useStashed)
+            })
+        },
+        // this samizdatFetch() will run first
+        // all other promises generated by SamizdatPlugins[] will be chained on it
+        // using the catch() in reduce() above
+        samizdatFetch(SamizdatPlugins[0], url, null, useStashed)
+    )
+    .then((response)=>{
+        // yes, this returns a promise...
+        return SamizdatInfo.resources[url].method.then((method)=>{
+
+            // get the plugin that was used to fetch content
+            plugin = getSamizdatPluginByName(method)
+            
+            // if it's a stashing plugin...
+            if (typeof plugin.stash === 'function') {
+                // we obviously do not want to stash
+                console.log('(COMMIT_UNKNOWN) Not stashing, since resource is already retrieved by a stashing plugin:', url);
+                // since we got the data from a stashing plugin,
+                // let's run the rest of plugins in the background to check if we can get a fresher resource
+                // and stash it in cache for later use
+                console.log('(COMMIT_UNKNOWN) starting background no-stashed fetch for:', url);
+                getResourceThroughSamizdat(url, false, true)
+                // return the response so that stuff can keep happening
+                return response
+
+            // otherwise, let's see if we want to stash
+            } else if (doStash) {
+            
+                // find the first stashing plugin
+                for (i=0; i<SamizdatPlugins.length; i++) {
+                    if (typeof SamizdatPlugins[i].stash === 'function') {
+                        
+                        // ok, now we're in business
+                        console.log('(COMMIT_UNKNOWN) Stashing a successful fetch of:', url);
+                        console.log('+-- fetched using  :', method)
+                        console.log('+-- stashing using :', SamizdatPlugins[i].name)
+
+                        // working on clone()'ed response so that the original one is not touched
+                        // TODO: should a failed stashing break the flow here? probably not!
+                        return SamizdatPlugins[i].stash(response.clone(), url)
+                            .then((res)=>{
+                                // original response will be needed further down
+                                return response
+                            })
+                    }
+                }
+            }
+            // if we're here it means we went through the whole list of plugins 
+            // and found not a single stashing plugin
+            // or we don't want to stash the resources in the first place
+            // that's fine, but let's make sure the response goes forth
+            return response
+        })
+    })
+    // a final catch... in case all plugins fail
+    .catch((err)=>{
+        console.log("(COMMIT_UNKNOWN) Samizdat also failed completely: ", err);
+        console.log('+-- URL    : ' + url)
+        // cleanup
+        SamizdatInfo.resources[url].fetchError = err.toString()
+        // this is very naïve and should in fact be handled inside the relevant plugin, probably
+        SamizdatInfo.resources[url].method = null
+        // rethrow
+        throw err
+    })
+    // this is where we can attach a .then() that will run regardless of the result
+    // which could then:
+    // - cache the results or perform some other housekeeping in case of success
+    // - handle the error in case of failure
+}
+
+/* ========================================================================= *\
+|* === Setting up the event handlers                                     === *|
+\* ========================================================================= */
+
 self.addEventListener('install', event => {
     // TODO: Might we want to have a local cache?
     // "COMMIT_UNKNOWN" will be replaced with commit ID
@@ -178,76 +335,7 @@ self.addEventListener('fetch', event => {
     // clean the URL, removing any fragment identifier
     var cleanURL = event.request.url.replace(/#.+$/, '');
 
-    // make sure SamizdatInfo exists for the cleanURL
-    if (typeof SamizdatInfo.resources[cleanURL] === 'undefined') {
-        SamizdatInfo.resources[cleanURL] = new SamizdatResourceInfo(cleanURL)
-        console.log('+-- created SamizdatInfo.resources[' + cleanURL + ']')
-    }
-
-    // clear the error fetchError info
-    SamizdatInfo.resources[cleanURL].fetchError = null;
-    SamizdatInfo.resources[cleanURL].method = 'fetch'; // assume the standard fetch will work
-
     // GET requests to our own domain that are *not* #samizdat-info requests
     // get handled by plugins in case of an error
-    return void event.respondWith(
-        /**
-         * this uses Array.reduce() to chain the SamizdatPlugins[]-generated Promises
-         * using the Promise from a basic HTTP(S) fetch() as the default value
-         * 
-         * see: https://css-tricks.com/why-using-reduce-to-sequentially-resolve-promises-works/
-         * 
-         * this also means that fetch() below will run first (counter-intutively!)
-         */
-        SamizdatPlugins.reduce((prevPromise, nextPlugin)=>{
-            return prevPromise.catch((error)=>{
-                // Fall back to Samizdat:
-                console.log("(COMMIT_UNKNOWN) Samizdat fallback handling after: ", error);
-                console.log('+-- URL    : ' + cleanURL)
-                console.log('+-- method : ' + nextPlugin.name)
-                // record the error in SamizdatInfo (as a string)
-                SamizdatInfo.resources[cleanURL].fetchError = error.toString()
-                // this is very naïve and should in fact be handled inside the relevant plugin, probably
-                SamizdatInfo.resources[cleanURL].method = nextPlugin.name;
-                // use the plugin
-                return nextPlugin.fetch(cleanURL)
-            })
-        // this fetch() will run first
-        // all other promises generated by SamizdatPlugins[] will be chained using catch() on it
-        }, fetch(event.request)
-            .then((response) => {
-                // that's a paddlin'
-                if (response.status >= 400) {
-                    // throw an Error to fall back to Samizdat:
-                    throw new Error('HTTP Error: ' + response.status + ' ' + response.statusText);
-                }
-                // all good, it seems
-                console.log("Fetched:", response.url);
-                return response;
-            })
-        )
-        /*.then((response)=>{
-            // TODO: this assumes the cache plugin is the first one!
-            // TODO: we should not be caching a response fetched from cache!
-            console.log("(COMMIT_UNKNOWN) Caching successful fetch: ", cleanURL);
-            return SamizdatPlugins[0].push(response.clone(), cleanURL).then((res)=>{
-                return response
-            })
-        })*/
-        // a final catch... in case all plugins fail
-        .catch((err)=>{
-            console.log("(COMMIT_UNKNOWN) Samizdat also failed completely: ", err);
-            console.log('+-- URL    : ' + cleanURL)
-            // cleanup
-            SamizdatInfo.resources[cleanURL].fetchError = err.toString()
-            // this is very naïve and should in fact be handled inside the relevant plugin, probably
-            SamizdatInfo.resources[cleanURL].method = null
-            // rethrow
-            throw err
-        })
-        // this is where we can attach a .then() that will run regardless of the result
-        // which could then:
-        // - cache the results or perform some other housekeeping in case of success
-        // - handle the error in case of failure
-    );
+    return void event.respondWith(getResourceThroughSamizdat(cleanURL))
 });