diff --git a/.eslintignore b/.eslintignore index 9b27377..4ced8a0 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1,3 @@ testpilot-metrics.js +lib/shield/*.js +lib/testpilot/*.js diff --git a/README.md b/README.md index 3fead42..dd1db0d 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ -# Containers: Test Pilot Experiment +# Containers Add-on [![Available on Test Pilot](https://img.shields.io/badge/available_on-Test_Pilot-0996F8.svg)](https://testpilot.firefox.com/experiments/containers) -[Embedded Web Extension](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Embedded_WebExtensions) to experiment with [Containers](https://blog.mozilla.org/tanvi/2016/06/16/contextual-identities-on-the-web/) in [Firefox Test Pilot](https://testpilot.firefox.com/) to learn: +[Embedded Web Extension](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Embedded_WebExtensions) to build [Containers](https://blog.mozilla.org/tanvi/2016/06/16/contextual-identities-on-the-web/) as a Firefox [Test Pilot](https://testpilot.firefox.com/) Experiment and [Shield Study](https://wiki.mozilla.org/Firefox/Shield/Shield_Studies) to learn: * Will a general Firefox audience understand the Containers feature? * Is the UI as currently implemented in Nightly clear or discoverable? -See [the Product Hypothesis Document for more -details](https://docs.google.com/document/d/1WQdHTVXROk7dYkSFluc6_hS44tqZjIrG9I-uPyzevE8/edit?ts=5824ba12#). +For more info, see: + +* [Test Pilot Product Hypothesis Document](https://docs.google.com/document/d/1WQdHTVXROk7dYkSFluc6_hS44tqZjIrG9I-uPyzevE8/edit#) +* [Shield Product Hypothesis Document](https://docs.google.com/document/d/1vMD-fH_5hGDDqNvpRZk12_RhCN2WAe4_yaBamaNdtik/edit#) ## Requirements @@ -17,38 +19,28 @@ details](https://docs.google.com/document/d/1WQdHTVXROk7dYkSFluc6_hS44tqZjIrG9I- * Firefox 51+ -## Run it - -See Development - - ## Development ### Development Environment Add-on development is better with [a particular environment](https://developer.mozilla.org/en-US/Add-ons/Setting_up_extension_development_environment). One simple way to get that environment set up is to install the [DevPrefs add-on](https://addons.mozilla.org/en-US/firefox/addon/devprefs/). You can make a custom Firefox profile that includes the DevPrefs add-on, and use that profile when you run the code in this repository. - 1. Make a new profile by running `/path/to/firefox -P`, which launches the profile editor. "Create Profile" -- name it whatever you wish (e.g. 'addon_dev') and store it in the default location. It's probably best to deselect the option to "Use without asking," since you probably don't want to use this as your default profile. 2. Once you've created your profile, click "Start Firefox". A new instance of Firefox should launch. Go to Tools->Add-ons and search for "DevPrefs". Install it. Quit Firefox. 3. Now you have a new, vanilla Firefox profile with the DevPrefs add-on installed. You can use your new profile with the code in _this_ repository like so: -**Beta building** +#### Run the `.xpi` file in an unbranded build +Release & Beta channels do not allow un-signed add-ons, even with the DevPrefs. So, you must run the add-on in an [unbranded build](https://wiki.mozilla.org/Add-ons/Extension_Signing#Unbranded_Builds): -To build this for 51 beta just using the downloaded version of beta will not work as XPI signature checking is disabled fully. +1. Download and install an un-branded build of Firefox +2. Download the latest `.xpi` from this repository's releases +3. Run the un-branded build of Firefox with your DevPrefs profile +4. Go to `about:addons` +5. Click the gear, and select "Install Add-on From File..." +6. Select the `.xpi` file -The only way to run the experiment is using an [unbranded version build](https://wiki.mozilla.org/Add-ons/Extension_Signing#Unbranded_Builds) or to build beta yourself: - -1. [Download the mozilla-beta repo](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial#mozilla-beta_(prerelease_development_tree)) -2. [Create a mozconfig file](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Build_Instructions/Configuring_Build_Options) - probably optional -3. `cd ` -3. `./mach bootstrap` -4. `./mach build` -5. Follow the above instructions by creating the new profile via: `~//obj-x86_64-pc-linux-gnu/dist/bin/firefox -P` (Where "obj-x86_64-pc-linux-gnu" may be different depending on platform obj-...) - - -### Run with jpm +#### Run the TxP experiment with `jpm` 1. `git clone git@github.com:mozilla/testpilot-containers.git` 2. `cd testpilot-containers` @@ -57,11 +49,22 @@ The only way to run the experiment is using an [unbranded version build](https:/ Check out the [Browser Toolbox](https://developer.mozilla.org/en-US/docs/Tools/Browser_Toolbox) for more information about debugging add-on code. +#### Run the shield study with `shield` + +1. `git clone git@github.com:mozilla/testpilot-containers.git` +2. `cd testpilot-containers` +3. `npm install -g shield-study-cli` +4. `shield run . -- --binary Nightly` + ### Building .xpi -To build a local .xpi, use the plain [`jpm -xpi`](https://developer.mozilla.org/en-US/Add-ons/SDK/Tools/jpm#jpm_xpi) command. +To build a local testpilot-containers.xpi, use the plain [`jpm +xpi`](https://developer.mozilla.org/en-US/Add-ons/SDK/Tools/jpm#jpm_xpi) command, +or run `npm run build`. + +#### Building a shield .xpi +To build a local shield-study-containers.xpi, run `npm run build-shield`. ### Signing an .xpi diff --git a/index.js b/index.js index bf70a1c..e9f57be 100644 --- a/index.js +++ b/index.js @@ -60,6 +60,7 @@ const prefService = require("sdk/preferences/service"); const self = require("sdk/self"); const { Services } = require("resource://gre/modules/Services.jsm"); const ss = require("sdk/simple-storage"); +const { study } = require("./study"); const { Style } = require("sdk/stylesheet/style"); const tabs = require("sdk/tabs"); const tabsUtils = require("sdk/tabs/utils"); @@ -320,6 +321,8 @@ const ContainerService = { // End-Of-Hack Services.obs.addObserver(this, "lightweight-theme-changed", false); + + study.startup(reason); }, registerBackgroundConnection(api) { diff --git a/lib/shield/event-target.js b/lib/shield/event-target.js new file mode 100644 index 0000000..4335d8c --- /dev/null +++ b/lib/shield/event-target.js @@ -0,0 +1,55 @@ +/** + * Drop-in replacement for {@link external:sdk/event/target.EventTarget} for use + * with es6 classes. + * @module event-target + * @author Martin Giger + * @license MPL-2.0 + */ + /** + * An SDK class that add event reqistration methods + * @external sdk/event/target + * @requires sdk/event/target + */ +/** + * @class EventTarget + * @memberof external:sdk/event/target + * @see {@link https://developer.mozilla.org/en-US/Add-ons/SDK/Low-Level_APIs/event_target#EventTarget} + */ + +// slightly modified from: https://raw.githubusercontent.com/freaktechnik/justintv-stream-notifications/master/lib/event-target.js + +"use strict"; + +const { on, once, off, setListeners } = require("sdk/event/core"); + +/* istanbul ignore next */ +/** + * @class + */ +class EventTarget { + constructor(options) { + setListeners(this, options); + } + + on(...args) { + on(this, ...args); + return this; + } + + once(...args) { + once(this, ...args); + return this; + } + + off(...args) { + off(this, ...args); + return this; + } + + removeListener(...args) { + off(this, ...args); + return this; + } +} + +exports.EventTarget = EventTarget; diff --git a/lib/shield/index.js b/lib/shield/index.js new file mode 100644 index 0000000..9d4ec95 --- /dev/null +++ b/lib/shield/index.js @@ -0,0 +1,428 @@ +"use strict"; + +// Chrome privileged +const {Cu} = require("chrome"); +const { Services } = Cu.import("resource://gre/modules/Services.jsm"); +const { TelemetryController } = Cu.import("resource://gre/modules/TelemetryController.jsm"); +const CID = Cu.import("resource://gre/modules/ClientID.jsm"); + +// sdk +const { merge } = require("sdk/util/object"); +const querystring = require("sdk/querystring"); +const { prefs } = require("sdk/simple-prefs"); +const prefSvc = require("sdk/preferences/service"); +const { setInterval } = require("sdk/timers"); +const tabs = require("sdk/tabs"); +const { URL } = require("sdk/url"); + +const { EventTarget } = require("./event-target"); +const { emit } = require("sdk/event/core"); +const self = require("sdk/self"); + +const DAY = 86400*1000; + +// ongoing within-addon fuses / timers +let lastDailyPing = Date.now(); + +/* Functional, self-contained utils */ + +// equal probability choices from a list "choices" +function chooseVariation(choices,rng=Math.random()) { + let l = choices.length; + return choices[Math.floor(l*Math.random())]; +} + +function dateToUTC(date) { + return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours(), date.getUTCMinutes(), date.getUTCSeconds()); +} + +function generateTelemetryIdIfNeeded() { + let id = TelemetryController.clientID; + /* istanbul ignore next */ + if (id == undefined) { + return CID.ClientIDImpl._doLoadClientID() + } else { + return Promise.resolve(id) + } +} + +function userId () { + return prefSvc.get("toolkit.telemetry.cachedClientID","unknown"); +} + +var Reporter = new EventTarget().on("report", + (d) => prefSvc.get('shield.debug') && console.log("report",d) +); + +function report(data, src="addon", bucket="shield-study") { + data = merge({}, data , { + study_version: self.version, + about: { + _src: src, + _v: 2 + } + }); + if (prefSvc.get('shield.testing')) data.testing = true + + emit(Reporter, "report", data); + let telOptions = {addClientId: true, addEnvironment: true} + return TelemetryController.submitExternalPing(bucket, data, telOptions); +} + +function survey (url, queryArgs={}) { + if (! url) return + + let U = new URL(url); + let q = U.search; + if (q) { + url = U.href.split(q)[0]; + q = querystring.parse(querystring.unescape(q.slice(1))); + } else { + q = {}; + } + // get user info. + let newArgs = merge({}, + q, + queryArgs + ); + let searchstring = querystring.stringify(newArgs); + url = url + "?" + searchstring; + return url; +} + + +function setOrGetFirstrun () { + let firstrun = prefs["shield.firstrun"]; + if (firstrun === undefined) { + firstrun = prefs["shield.firstrun"] = String(dateToUTC(new Date())) // in utc, user set + } + return Number(firstrun) +} + +function reuseVariation (choices) { + return prefs["shield.variation"]; +} + +function setVariation (choice) { + prefs["shield.variation"] = choice + return choice +} + +function die (addonId=self.id) { + /* istanbul ignore else */ + if (prefSvc.get("shield.fakedie")) return; + /* istanbul ignore next */ + require("sdk/addon/installer").uninstall(addonId); +} + +// TODO: GRL vulnerable to clock time issues #1 +function expired (xconfig, now = Date.now() ) { + return ((now - Number(xconfig.firstrun))/ DAY) > xconfig.days; +} + +function resetShieldPrefs () { + delete prefs['shield.firstrun']; + delete prefs['shield.variation']; +} + +function cleanup () { + prefSvc.keys(`extensions.${self.preferencesBranch}`).forEach ( + (p) => { + delete prefs[p]; + }) +} + +function telemetrySubset (xconfig) { + return { + study_name: xconfig.name, + branch: xconfig.variation, + } +} + +class Study extends EventTarget { + constructor (config) { + super(); + this.config = merge({ + name: self.addonId, + variations: {'observe-only': () => {}}, + surveyUrls: {}, + days: 7 + },config); + + this.config.firstrun = setOrGetFirstrun(); + + let variation = reuseVariation(); + if (variation === undefined) { + variation = this.decideVariation(); + if (!(variation in this.config.variations)) { + // chaijs doesn't think this is an instanceof Error + // freaktechnik and gregglind debugged for a while. + // sdk errors might not be 'Errors' or chai is wack, who knows. + // https://dxr.mozilla.org/mozilla-central/search?q=regexp%3AError%5Cs%3F(%3A%7C%3D)+path%3Aaddon-sdk%2Fsource%2F&redirect=false would list + throw new Error("Study Error: chosen variation must be in config.variations") + } + setVariation(variation); + } + this.config.variation = variation; + + this.flags = { + ineligibleDie: undefined + }; + this.states = []; + // all these work, but could be cleaner. I hate the `bind` stuff. + this.on( + "change", (function (newstate) { + prefSvc.get('shield.debug') && console.log(newstate, this.states); + this.states.push(newstate); + emit(this, newstate); // could have checks here. + }).bind(this) + ) + this.on( + "starting", (function () { + this.changeState("modifying"); + }).bind(this) + ) + this.on( + "maybe-installing", (function () { + if (!this.isEligible()) { + this.changeState("ineligible-die"); + } else { + this.changeState("installed") + } + }).bind(this) + ) + this.on( + "ineligible-die", (function () { + try {this.whenIneligible()} catch (err) {/*ok*/} finally { /*ok*/ } + this.flags.ineligibleDie = true; + this.report(merge({}, telemetrySubset(this.config), {study_state: "ineligible"}), "shield"); + this.final(); + die(); + }).bind(this) + ) + this.on( + "installed", (function () { + try {this.whenInstalled()} catch (err) {/*ok*/} finally { /*ok*/ } + this.report(merge({}, telemetrySubset(this.config), {study_state: "install"}), "shield"); + this.changeState("modifying"); + }).bind(this) + ) + this.on( + "modifying", (function () { + var mybranchname = this.variation; + this.config.variations[mybranchname](); // do the effect + this.changeState("running"); + }).bind(this) + ) + this.on( // the one 'many' + "running", (function () { + // report success + this.report(merge({}, telemetrySubset(this.config), {study_state: "running"}), "shield"); + this.final(); + }).bind(this) + ) + this.on( + "normal-shutdown", (function () { + this.flags.dying = true; + this.report(merge({}, telemetrySubset(this.config), {study_state: "shutdown"}), "shield"); + this.final(); + }).bind(this) + ) + this.on( + "end-of-study", (function () { + if (this.flags.expired) { // safe to call multiple times + this.final(); + return; + } else { + // first time seen. + this.flags.expired = true; + try {this.whenComplete()} catch (err) { /*ok*/ } finally { /*ok*/ } + this.report(merge({}, telemetrySubset(this.config) ,{study_state: "end-of-study"}), "shield"); + // survey for end of study + let that = this; + generateTelemetryIdIfNeeded().then(()=>that.showSurvey("end-of-study")); + try {this.cleanup()} catch (err) {/*ok*/} finally { /*ok*/ } + this.final(); + die(); + } + }).bind(this) + ) + this.on( + "user-uninstall-disable", (function () { + if (this.flags.dying) { + this.final(); + return; + } + this.flags.dying = true; + this.report(merge({}, telemetrySubset(this.config), {study_state: "user-ended-study"}), "shield"); + let that = this; + generateTelemetryIdIfNeeded().then(()=>that.showSurvey("user-ended-study")); + try {this.cleanup()} catch (err) {/*ok*/} finally { /*ok*/ } + this.final(); + die(); + }).bind(this) + ) + } + + get state () { + let n = this.states.length; + return n ? this.states[n-1] : undefined + } + + get variation () { + return this.config.variation; + } + + get firstrun () { + return this.config.firstrun; + } + + dieIfExpired () { + let xconfig = this.config; + if (expired(xconfig)) { + emit(this, "change", "end-of-study"); + return true + } else { + return false + } + } + + alivenessPulse (last=lastDailyPing) { + // check for new day, phone home if true. + let t = Date.now(); + if ((t - last) >= DAY) { + lastDailyPing = t; + // phone home + emit(this,"change","running"); + } + // check expiration, and die with report if needed + return this.dieIfExpired(); + } + + changeState (newstate) { + emit(this,'change', newstate); + } + + final () { + emit(this,'final', {}); + } + + startup (reason) { + // https://developer.mozilla.org/en-US/Add-ons/SDK/Tutorials/Listening_for_load_and_unload + + // check expiry first, before anything, quit and die if so + + // check once, right away, short circuit both install and startup + // to prevent modifications from happening. + if (this.dieIfExpired()) return this + + switch (reason) { + case "install": + emit(this, "change", "maybe-installing"); + break; + + case "enable": + case "startup": + case "upgrade": + case "downgrade": + emit(this, "change", "starting"); + } + + if (! this._pulseTimer) this._pulseTimer = setInterval(this.alivenessPulse.bind(this), 5*60*1000 /*5 minutes */) + return this; + } + + shutdown (reason) { + // https://developer.mozilla.org/en-US/Add-ons/SDK/Tutorials/Listening_for_load_and_unload + if (this.flags.ineligibleDie || + this.flags.expired || + this.flags.dying + ) { return this } // special cases. + + switch (reason) { + case "uninstall": + case "disable": + emit(this, "change", "user-uninstall-disable"); + break; + + // 5. usual end of session. + case "shutdown": + case "upgrade": + case "downgrade": + emit(this, "change", "normal-shutdown") + break; + } + return this; + } + + cleanup () { + // do the simple prefs and simplestorage cleanup + // extend by extension + resetShieldPrefs(); + cleanup(); + } + + isEligible () { + return true; + } + + whenIneligible () { + // empty function unless overrided + } + + whenInstalled () { + // empty unless overrided + } + + whenComplete () { + // when the study expires + } + + /** + * equal choice from varations, by default. override to get unequal + */ + decideVariation (rng=Math.random()) { + return chooseVariation(Object.keys(this.config.variations), rng); + } + + get surveyQueryArgs () { + return { + variation: this.variation, + xname: this.config.name, + who: userId(), + updateChannel: Services.appinfo.defaultUpdateChannel, + fxVersion: Services.appinfo.version, + } + } + + showSurvey(reason) { + let partial = this.config.surveyUrls[reason]; + + let queryArgs = this.surveyQueryArgs; + queryArgs.reason = reason; + if (partial) { + let url = survey(partial, queryArgs); + tabs.open(url); + return url + } else { + return + } + } + + report () { // convenience only + return report.apply(null, arguments); + } +} + +module.exports = { + chooseVariation: chooseVariation, + die: die, + expired: expired, + generateTelemetryIdIfNeeded: generateTelemetryIdIfNeeded, + report: report, + Reporter: Reporter, + resetShieldPrefs: resetShieldPrefs, + Study: Study, + cleanup: cleanup, + survey: survey +} diff --git a/lib/testpilot/experiment.js b/lib/testpilot/experiment.js new file mode 100644 index 0000000..1b4e98f --- /dev/null +++ b/lib/testpilot/experiment.js @@ -0,0 +1,95 @@ +const { AddonManager } = require('resource://gre/modules/AddonManager.jsm'); +const { ClientID } = require('resource://gre/modules/ClientID.jsm'); +const Events = require('sdk/system/events'); +const { Services } = require('resource://gre/modules/Services.jsm'); +const { storage } = require('sdk/simple-storage'); +const { + TelemetryController +} = require('resource://gre/modules/TelemetryController.jsm'); +const { Request } = require('sdk/request'); + + +const EVENT_SEND_METRIC = 'testpilot::send-metric'; +const startTime = (Services.startup.getStartupInfo().process); + +function makeTimestamp(timestamp) { + return Math.round((timestamp - startTime) / 1000); +} + +function experimentPing(event) { + const timestamp = new Date(); + const { subject, data } = event; + let parsed; + try { + parsed = JSON.parse(data); + } catch (err) { + // eslint-disable-next-line no-console + return console.error(`Dropping bad metrics packet: ${err}`); + } + + AddonManager.getAddonByID(subject, addon => { + const payload = { + test: subject, + version: addon.version, + timestamp: makeTimestamp(timestamp), + variants: storage.experimentVariants && + subject in storage.experimentVariants + ? storage.experimentVariants[subject] + : null, + payload: parsed + }; + TelemetryController.submitExternalPing('testpilottest', payload, { + addClientId: true, + addEnvironment: true + }); + + // TODO: DRY up this ping centre code here and in lib/Telemetry. + const pcPing = TelemetryController.getCurrentPingData(); + pcPing.type = 'testpilot'; + pcPing.payload = payload; + const pcPayload = { + // 'method' is used by testpilot-metrics library. + // 'event' was used before that library existed. + event_type: parsed.event || parsed.method, + client_time: makeTimestamp(parsed.timestamp || timestamp), + addon_id: subject, + addon_version: addon.version, + firefox_version: pcPing.environment.build.version, + os_name: pcPing.environment.system.os.name, + os_version: pcPing.environment.system.os.version, + locale: pcPing.environment.settings.locale, + // Note: these two keys are normally inserted by the ping-centre client. + client_id: ClientID.getCachedClientID(), + topic: 'testpilot' + }; + // Add any other extra top-level keys = require(the payload, possibly including + // 'object' or 'category', among others. + Object.keys(parsed).forEach(f => { + // Ignore the keys we've already added to `pcPayload`. + const ignored = ['event', 'method', 'timestamp']; + if (!ignored.includes(f)) { + pcPayload[f] = parsed[f]; + } + }); + + const req = new Request({ + url: 'https://tiles.services.mozilla.com/v3/links/ping-centre', + contentType: 'application/json', + content: JSON.stringify(pcPayload) + }); + req.post(); + }); +} + +function Experiment() { + // If the user has @testpilot-addon, it already bound + // experimentPing to testpilot::send-metric, + // so we don't need to bind this one + AddonManager.getAddonByID('@testpilot-addon', addon => { + if (!addon) { + Events.on(EVENT_SEND_METRIC, experimentPing); + } + }); +} + +module.exports = Experiment; diff --git a/package.json b/package.json index 020a9b4..f910d9c 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,9 @@ "eslint-plugin-promise": "^3.4.0", "htmllint-cli": "^0.0.5", "jpm": "^1.2.2", + "json": "^9.0.6", "npm-run-all": "^4.0.0", + "shield-studies-addon-utils": "^2.0.0", "stylelint": "^7.9.0", "stylelint-config-standard": "^16.0.0", "stylelint-order": "^0.3.0", @@ -41,6 +43,7 @@ }, "scripts": { "build": "npm test && jpm xpi", + "build-shield": "npm test && json -I -f package.json -e 'this.name=\"shield-study-containers\"' && jpm xpi && json -I -f package.json -e 'this.name=\"testpilot-containers\"'", "deploy": "deploy-txp", "lint": "npm-run-all lint:*", "lint:addon": "addons-linter webextension --self-hosted", diff --git a/study.js b/study.js new file mode 100644 index 0000000..672e5f6 --- /dev/null +++ b/study.js @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const self = require("sdk/self"); +const { when: unload } = require("sdk/system/unload"); +const tabs = require("sdk/tabs"); + +const shield = require("./lib/shield/index"); + +const studyConfig = { + name: self.addonId, + days: 28, + surveyUrls: { + }, + variations: { + "control": () => {}, + "privacyOnboarding": () => {}, + "onlineAccountsOnboarding": () => {}, + "tabManagementOnboarding": () => {} + } +}; + +class ContainersStudy extends shield.Study { + isEligible () { + // If the user already has testpilot-containers extension, they are in the + // Test Pilot experiment, so exclude them. + return super.isEligible(); + } + + whenEligible () { + } + + whenInstalled () { + tabs.open(`data:text/html, Thank you for helping us study Containers in Firefox. You are in the ${this.variation} variation.`); + } + + cleanup() { + } +} + +const thisStudy = new ContainersStudy(studyConfig); + +unload((reason) => thisStudy.shutdown(reason)); + +exports.study = thisStudy; diff --git a/testpilot-metrics.js b/testpilot-metrics.js index f68aae3..2914884 100644 --- a/testpilot-metrics.js +++ b/testpilot-metrics.js @@ -1,6 +1,9 @@ // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. +const Experiment = require('./lib/testpilot/experiment'); + +const experiment = new Experiment(); /** * Class that represents a metrics event broker. Events are sent to Google