Merge pull request #478 from mozilla/shield

Add lib/shield to enable shield study
This commit is contained in:
luke crouch 2017-05-25 09:16:15 -05:00 committed by GitHub
commit 2d26f95cb7
9 changed files with 663 additions and 25 deletions

View file

@ -1 +1,3 @@
testpilot-metrics.js
lib/shield/*.js
lib/testpilot/*.js

View file

@ -1,14 +1,16 @@
# Containers: Test Pilot Experiment
# Containers Add-on
[![Available on Test Pilot](https://img.shields.io/badge/available_on-Test_Pilot-0996F8.svg)](https://testpilot.firefox.com/experiments/containers)
[Embedded Web Extension](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Embedded_WebExtensions) to experiment with [Containers](https://blog.mozilla.org/tanvi/2016/06/16/contextual-identities-on-the-web/) in [Firefox Test Pilot](https://testpilot.firefox.com/) to learn:
[Embedded Web Extension](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Embedded_WebExtensions) to build [Containers](https://blog.mozilla.org/tanvi/2016/06/16/contextual-identities-on-the-web/) as a Firefox [Test Pilot](https://testpilot.firefox.com/) Experiment and [Shield Study](https://wiki.mozilla.org/Firefox/Shield/Shield_Studies) to learn:
* Will a general Firefox audience understand the Containers feature?
* Is the UI as currently implemented in Nightly clear or discoverable?
See [the Product Hypothesis Document for more
details](https://docs.google.com/document/d/1WQdHTVXROk7dYkSFluc6_hS44tqZjIrG9I-uPyzevE8/edit?ts=5824ba12#).
For more info, see:
* [Test Pilot Product Hypothesis Document](https://docs.google.com/document/d/1WQdHTVXROk7dYkSFluc6_hS44tqZjIrG9I-uPyzevE8/edit#)
* [Shield Product Hypothesis Document](https://docs.google.com/document/d/1vMD-fH_5hGDDqNvpRZk12_RhCN2WAe4_yaBamaNdtik/edit#)
## Requirements
@ -17,38 +19,28 @@ details](https://docs.google.com/document/d/1WQdHTVXROk7dYkSFluc6_hS44tqZjIrG9I-
* Firefox 51+
## Run it
See Development
## Development
### Development Environment
Add-on development is better with [a particular environment](https://developer.mozilla.org/en-US/Add-ons/Setting_up_extension_development_environment). One simple way to get that environment set up is to install the [DevPrefs add-on](https://addons.mozilla.org/en-US/firefox/addon/devprefs/). You can make a custom Firefox profile that includes the DevPrefs add-on, and use that profile when you run the code in this repository.
1. Make a new profile by running `/path/to/firefox -P`, which launches the profile editor. "Create Profile" -- name it whatever you wish (e.g. 'addon_dev') and store it in the default location. It's probably best to deselect the option to "Use without asking," since you probably don't want to use this as your default profile.
2. Once you've created your profile, click "Start Firefox". A new instance of Firefox should launch. Go to Tools->Add-ons and search for "DevPrefs". Install it. Quit Firefox.
3. Now you have a new, vanilla Firefox profile with the DevPrefs add-on installed. You can use your new profile with the code in _this_ repository like so:
**Beta building**
#### Run the `.xpi` file in an unbranded build
Release & Beta channels do not allow un-signed add-ons, even with the DevPrefs. So, you must run the add-on in an [unbranded build](https://wiki.mozilla.org/Add-ons/Extension_Signing#Unbranded_Builds):
To build this for 51 beta just using the downloaded version of beta will not work as XPI signature checking is disabled fully.
1. Download and install an un-branded build of Firefox
2. Download the latest `.xpi` from this repository's releases
3. Run the un-branded build of Firefox with your DevPrefs profile
4. Go to `about:addons`
5. Click the gear, and select "Install Add-on From File..."
6. Select the `.xpi` file
The only way to run the experiment is using an [unbranded version build](https://wiki.mozilla.org/Add-ons/Extension_Signing#Unbranded_Builds) or to build beta yourself:
1. [Download the mozilla-beta repo](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial#mozilla-beta_(prerelease_development_tree))
2. [Create a mozconfig file](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Build_Instructions/Configuring_Build_Options) - probably optional
3. `cd <reponame>`
3. `./mach bootstrap`
4. `./mach build`
5. Follow the above instructions by creating the new profile via: `~/<reponame>/obj-x86_64-pc-linux-gnu/dist/bin/firefox -P` (Where "obj-x86_64-pc-linux-gnu" may be different depending on platform obj-...)
### Run with jpm
#### Run the TxP experiment with `jpm`
1. `git clone git@github.com:mozilla/testpilot-containers.git`
2. `cd testpilot-containers`
@ -57,11 +49,22 @@ The only way to run the experiment is using an [unbranded version build](https:/
Check out the [Browser Toolbox](https://developer.mozilla.org/en-US/docs/Tools/Browser_Toolbox) for more information about debugging add-on code.
#### Run the shield study with `shield`
1. `git clone git@github.com:mozilla/testpilot-containers.git`
2. `cd testpilot-containers`
3. `npm install -g shield-study-cli`
4. `shield run . -- --binary Nightly`
### Building .xpi
To build a local .xpi, use the plain [`jpm
xpi`](https://developer.mozilla.org/en-US/Add-ons/SDK/Tools/jpm#jpm_xpi) command.
To build a local testpilot-containers.xpi, use the plain [`jpm
xpi`](https://developer.mozilla.org/en-US/Add-ons/SDK/Tools/jpm#jpm_xpi) command,
or run `npm run build`.
#### Building a shield .xpi
To build a local shield-study-containers.xpi, run `npm run build-shield`.
### Signing an .xpi

View file

@ -60,6 +60,7 @@ const prefService = require("sdk/preferences/service");
const self = require("sdk/self");
const { Services } = require("resource://gre/modules/Services.jsm");
const ss = require("sdk/simple-storage");
const { study } = require("./study");
const { Style } = require("sdk/stylesheet/style");
const tabs = require("sdk/tabs");
const tabsUtils = require("sdk/tabs/utils");
@ -320,6 +321,8 @@ const ContainerService = {
// End-Of-Hack
Services.obs.addObserver(this, "lightweight-theme-changed", false);
study.startup(reason);
},
registerBackgroundConnection(api) {

View file

@ -0,0 +1,55 @@
/**
* Drop-in replacement for {@link external:sdk/event/target.EventTarget} for use
* with es6 classes.
* @module event-target
* @author Martin Giger
* @license MPL-2.0
*/
/**
* An SDK class that add event reqistration methods
* @external sdk/event/target
* @requires sdk/event/target
*/
/**
* @class EventTarget
* @memberof external:sdk/event/target
* @see {@link https://developer.mozilla.org/en-US/Add-ons/SDK/Low-Level_APIs/event_target#EventTarget}
*/
// slightly modified from: https://raw.githubusercontent.com/freaktechnik/justintv-stream-notifications/master/lib/event-target.js
"use strict";
const { on, once, off, setListeners } = require("sdk/event/core");
/* istanbul ignore next */
/**
* @class
*/
class EventTarget {
constructor(options) {
setListeners(this, options);
}
on(...args) {
on(this, ...args);
return this;
}
once(...args) {
once(this, ...args);
return this;
}
off(...args) {
off(this, ...args);
return this;
}
removeListener(...args) {
off(this, ...args);
return this;
}
}
exports.EventTarget = EventTarget;

428
lib/shield/index.js Normal file
View file

@ -0,0 +1,428 @@
"use strict";
// Chrome privileged
const {Cu} = require("chrome");
const { Services } = Cu.import("resource://gre/modules/Services.jsm");
const { TelemetryController } = Cu.import("resource://gre/modules/TelemetryController.jsm");
const CID = Cu.import("resource://gre/modules/ClientID.jsm");
// sdk
const { merge } = require("sdk/util/object");
const querystring = require("sdk/querystring");
const { prefs } = require("sdk/simple-prefs");
const prefSvc = require("sdk/preferences/service");
const { setInterval } = require("sdk/timers");
const tabs = require("sdk/tabs");
const { URL } = require("sdk/url");
const { EventTarget } = require("./event-target");
const { emit } = require("sdk/event/core");
const self = require("sdk/self");
const DAY = 86400*1000;
// ongoing within-addon fuses / timers
let lastDailyPing = Date.now();
/* Functional, self-contained utils */
// equal probability choices from a list "choices"
function chooseVariation(choices,rng=Math.random()) {
let l = choices.length;
return choices[Math.floor(l*Math.random())];
}
function dateToUTC(date) {
return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), date.getUTCHours(), date.getUTCMinutes(), date.getUTCSeconds());
}
function generateTelemetryIdIfNeeded() {
let id = TelemetryController.clientID;
/* istanbul ignore next */
if (id == undefined) {
return CID.ClientIDImpl._doLoadClientID()
} else {
return Promise.resolve(id)
}
}
function userId () {
return prefSvc.get("toolkit.telemetry.cachedClientID","unknown");
}
var Reporter = new EventTarget().on("report",
(d) => prefSvc.get('shield.debug') && console.log("report",d)
);
function report(data, src="addon", bucket="shield-study") {
data = merge({}, data , {
study_version: self.version,
about: {
_src: src,
_v: 2
}
});
if (prefSvc.get('shield.testing')) data.testing = true
emit(Reporter, "report", data);
let telOptions = {addClientId: true, addEnvironment: true}
return TelemetryController.submitExternalPing(bucket, data, telOptions);
}
function survey (url, queryArgs={}) {
if (! url) return
let U = new URL(url);
let q = U.search;
if (q) {
url = U.href.split(q)[0];
q = querystring.parse(querystring.unescape(q.slice(1)));
} else {
q = {};
}
// get user info.
let newArgs = merge({},
q,
queryArgs
);
let searchstring = querystring.stringify(newArgs);
url = url + "?" + searchstring;
return url;
}
function setOrGetFirstrun () {
let firstrun = prefs["shield.firstrun"];
if (firstrun === undefined) {
firstrun = prefs["shield.firstrun"] = String(dateToUTC(new Date())) // in utc, user set
}
return Number(firstrun)
}
function reuseVariation (choices) {
return prefs["shield.variation"];
}
function setVariation (choice) {
prefs["shield.variation"] = choice
return choice
}
function die (addonId=self.id) {
/* istanbul ignore else */
if (prefSvc.get("shield.fakedie")) return;
/* istanbul ignore next */
require("sdk/addon/installer").uninstall(addonId);
}
// TODO: GRL vulnerable to clock time issues #1
function expired (xconfig, now = Date.now() ) {
return ((now - Number(xconfig.firstrun))/ DAY) > xconfig.days;
}
function resetShieldPrefs () {
delete prefs['shield.firstrun'];
delete prefs['shield.variation'];
}
function cleanup () {
prefSvc.keys(`extensions.${self.preferencesBranch}`).forEach (
(p) => {
delete prefs[p];
})
}
function telemetrySubset (xconfig) {
return {
study_name: xconfig.name,
branch: xconfig.variation,
}
}
class Study extends EventTarget {
constructor (config) {
super();
this.config = merge({
name: self.addonId,
variations: {'observe-only': () => {}},
surveyUrls: {},
days: 7
},config);
this.config.firstrun = setOrGetFirstrun();
let variation = reuseVariation();
if (variation === undefined) {
variation = this.decideVariation();
if (!(variation in this.config.variations)) {
// chaijs doesn't think this is an instanceof Error
// freaktechnik and gregglind debugged for a while.
// sdk errors might not be 'Errors' or chai is wack, who knows.
// https://dxr.mozilla.org/mozilla-central/search?q=regexp%3AError%5Cs%3F(%3A%7C%3D)+path%3Aaddon-sdk%2Fsource%2F&redirect=false would list
throw new Error("Study Error: chosen variation must be in config.variations")
}
setVariation(variation);
}
this.config.variation = variation;
this.flags = {
ineligibleDie: undefined
};
this.states = [];
// all these work, but could be cleaner. I hate the `bind` stuff.
this.on(
"change", (function (newstate) {
prefSvc.get('shield.debug') && console.log(newstate, this.states);
this.states.push(newstate);
emit(this, newstate); // could have checks here.
}).bind(this)
)
this.on(
"starting", (function () {
this.changeState("modifying");
}).bind(this)
)
this.on(
"maybe-installing", (function () {
if (!this.isEligible()) {
this.changeState("ineligible-die");
} else {
this.changeState("installed")
}
}).bind(this)
)
this.on(
"ineligible-die", (function () {
try {this.whenIneligible()} catch (err) {/*ok*/} finally { /*ok*/ }
this.flags.ineligibleDie = true;
this.report(merge({}, telemetrySubset(this.config), {study_state: "ineligible"}), "shield");
this.final();
die();
}).bind(this)
)
this.on(
"installed", (function () {
try {this.whenInstalled()} catch (err) {/*ok*/} finally { /*ok*/ }
this.report(merge({}, telemetrySubset(this.config), {study_state: "install"}), "shield");
this.changeState("modifying");
}).bind(this)
)
this.on(
"modifying", (function () {
var mybranchname = this.variation;
this.config.variations[mybranchname](); // do the effect
this.changeState("running");
}).bind(this)
)
this.on( // the one 'many'
"running", (function () {
// report success
this.report(merge({}, telemetrySubset(this.config), {study_state: "running"}), "shield");
this.final();
}).bind(this)
)
this.on(
"normal-shutdown", (function () {
this.flags.dying = true;
this.report(merge({}, telemetrySubset(this.config), {study_state: "shutdown"}), "shield");
this.final();
}).bind(this)
)
this.on(
"end-of-study", (function () {
if (this.flags.expired) { // safe to call multiple times
this.final();
return;
} else {
// first time seen.
this.flags.expired = true;
try {this.whenComplete()} catch (err) { /*ok*/ } finally { /*ok*/ }
this.report(merge({}, telemetrySubset(this.config) ,{study_state: "end-of-study"}), "shield");
// survey for end of study
let that = this;
generateTelemetryIdIfNeeded().then(()=>that.showSurvey("end-of-study"));
try {this.cleanup()} catch (err) {/*ok*/} finally { /*ok*/ }
this.final();
die();
}
}).bind(this)
)
this.on(
"user-uninstall-disable", (function () {
if (this.flags.dying) {
this.final();
return;
}
this.flags.dying = true;
this.report(merge({}, telemetrySubset(this.config), {study_state: "user-ended-study"}), "shield");
let that = this;
generateTelemetryIdIfNeeded().then(()=>that.showSurvey("user-ended-study"));
try {this.cleanup()} catch (err) {/*ok*/} finally { /*ok*/ }
this.final();
die();
}).bind(this)
)
}
get state () {
let n = this.states.length;
return n ? this.states[n-1] : undefined
}
get variation () {
return this.config.variation;
}
get firstrun () {
return this.config.firstrun;
}
dieIfExpired () {
let xconfig = this.config;
if (expired(xconfig)) {
emit(this, "change", "end-of-study");
return true
} else {
return false
}
}
alivenessPulse (last=lastDailyPing) {
// check for new day, phone home if true.
let t = Date.now();
if ((t - last) >= DAY) {
lastDailyPing = t;
// phone home
emit(this,"change","running");
}
// check expiration, and die with report if needed
return this.dieIfExpired();
}
changeState (newstate) {
emit(this,'change', newstate);
}
final () {
emit(this,'final', {});
}
startup (reason) {
// https://developer.mozilla.org/en-US/Add-ons/SDK/Tutorials/Listening_for_load_and_unload
// check expiry first, before anything, quit and die if so
// check once, right away, short circuit both install and startup
// to prevent modifications from happening.
if (this.dieIfExpired()) return this
switch (reason) {
case "install":
emit(this, "change", "maybe-installing");
break;
case "enable":
case "startup":
case "upgrade":
case "downgrade":
emit(this, "change", "starting");
}
if (! this._pulseTimer) this._pulseTimer = setInterval(this.alivenessPulse.bind(this), 5*60*1000 /*5 minutes */)
return this;
}
shutdown (reason) {
// https://developer.mozilla.org/en-US/Add-ons/SDK/Tutorials/Listening_for_load_and_unload
if (this.flags.ineligibleDie ||
this.flags.expired ||
this.flags.dying
) { return this } // special cases.
switch (reason) {
case "uninstall":
case "disable":
emit(this, "change", "user-uninstall-disable");
break;
// 5. usual end of session.
case "shutdown":
case "upgrade":
case "downgrade":
emit(this, "change", "normal-shutdown")
break;
}
return this;
}
cleanup () {
// do the simple prefs and simplestorage cleanup
// extend by extension
resetShieldPrefs();
cleanup();
}
isEligible () {
return true;
}
whenIneligible () {
// empty function unless overrided
}
whenInstalled () {
// empty unless overrided
}
whenComplete () {
// when the study expires
}
/**
* equal choice from varations, by default. override to get unequal
*/
decideVariation (rng=Math.random()) {
return chooseVariation(Object.keys(this.config.variations), rng);
}
get surveyQueryArgs () {
return {
variation: this.variation,
xname: this.config.name,
who: userId(),
updateChannel: Services.appinfo.defaultUpdateChannel,
fxVersion: Services.appinfo.version,
}
}
showSurvey(reason) {
let partial = this.config.surveyUrls[reason];
let queryArgs = this.surveyQueryArgs;
queryArgs.reason = reason;
if (partial) {
let url = survey(partial, queryArgs);
tabs.open(url);
return url
} else {
return
}
}
report () { // convenience only
return report.apply(null, arguments);
}
}
module.exports = {
chooseVariation: chooseVariation,
die: die,
expired: expired,
generateTelemetryIdIfNeeded: generateTelemetryIdIfNeeded,
report: report,
Reporter: Reporter,
resetShieldPrefs: resetShieldPrefs,
Study: Study,
cleanup: cleanup,
survey: survey
}

View file

@ -0,0 +1,95 @@
const { AddonManager } = require('resource://gre/modules/AddonManager.jsm');
const { ClientID } = require('resource://gre/modules/ClientID.jsm');
const Events = require('sdk/system/events');
const { Services } = require('resource://gre/modules/Services.jsm');
const { storage } = require('sdk/simple-storage');
const {
TelemetryController
} = require('resource://gre/modules/TelemetryController.jsm');
const { Request } = require('sdk/request');
const EVENT_SEND_METRIC = 'testpilot::send-metric';
const startTime = (Services.startup.getStartupInfo().process);
function makeTimestamp(timestamp) {
return Math.round((timestamp - startTime) / 1000);
}
function experimentPing(event) {
const timestamp = new Date();
const { subject, data } = event;
let parsed;
try {
parsed = JSON.parse(data);
} catch (err) {
// eslint-disable-next-line no-console
return console.error(`Dropping bad metrics packet: ${err}`);
}
AddonManager.getAddonByID(subject, addon => {
const payload = {
test: subject,
version: addon.version,
timestamp: makeTimestamp(timestamp),
variants: storage.experimentVariants &&
subject in storage.experimentVariants
? storage.experimentVariants[subject]
: null,
payload: parsed
};
TelemetryController.submitExternalPing('testpilottest', payload, {
addClientId: true,
addEnvironment: true
});
// TODO: DRY up this ping centre code here and in lib/Telemetry.
const pcPing = TelemetryController.getCurrentPingData();
pcPing.type = 'testpilot';
pcPing.payload = payload;
const pcPayload = {
// 'method' is used by testpilot-metrics library.
// 'event' was used before that library existed.
event_type: parsed.event || parsed.method,
client_time: makeTimestamp(parsed.timestamp || timestamp),
addon_id: subject,
addon_version: addon.version,
firefox_version: pcPing.environment.build.version,
os_name: pcPing.environment.system.os.name,
os_version: pcPing.environment.system.os.version,
locale: pcPing.environment.settings.locale,
// Note: these two keys are normally inserted by the ping-centre client.
client_id: ClientID.getCachedClientID(),
topic: 'testpilot'
};
// Add any other extra top-level keys = require(the payload, possibly including
// 'object' or 'category', among others.
Object.keys(parsed).forEach(f => {
// Ignore the keys we've already added to `pcPayload`.
const ignored = ['event', 'method', 'timestamp'];
if (!ignored.includes(f)) {
pcPayload[f] = parsed[f];
}
});
const req = new Request({
url: 'https://tiles.services.mozilla.com/v3/links/ping-centre',
contentType: 'application/json',
content: JSON.stringify(pcPayload)
});
req.post();
});
}
function Experiment() {
// If the user has @testpilot-addon, it already bound
// experimentPing to testpilot::send-metric,
// so we don't need to bind this one
AddonManager.getAddonByID('@testpilot-addon', addon => {
if (!addon) {
Events.on(EVENT_SEND_METRIC, experimentPing);
}
});
}
module.exports = Experiment;

View file

@ -16,7 +16,9 @@
"eslint-plugin-promise": "^3.4.0",
"htmllint-cli": "^0.0.5",
"jpm": "^1.2.2",
"json": "^9.0.6",
"npm-run-all": "^4.0.0",
"shield-studies-addon-utils": "^2.0.0",
"stylelint": "^7.9.0",
"stylelint-config-standard": "^16.0.0",
"stylelint-order": "^0.3.0",
@ -41,6 +43,7 @@
},
"scripts": {
"build": "npm test && jpm xpi",
"build-shield": "npm test && json -I -f package.json -e 'this.name=\"shield-study-containers\"' && jpm xpi && json -I -f package.json -e 'this.name=\"testpilot-containers\"'",
"deploy": "deploy-txp",
"lint": "npm-run-all lint:*",
"lint:addon": "addons-linter webextension --self-hosted",

46
study.js Normal file
View file

@ -0,0 +1,46 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
const self = require("sdk/self");
const { when: unload } = require("sdk/system/unload");
const tabs = require("sdk/tabs");
const shield = require("./lib/shield/index");
const studyConfig = {
name: self.addonId,
days: 28,
surveyUrls: {
},
variations: {
"control": () => {},
"privacyOnboarding": () => {},
"onlineAccountsOnboarding": () => {},
"tabManagementOnboarding": () => {}
}
};
class ContainersStudy extends shield.Study {
isEligible () {
// If the user already has testpilot-containers extension, they are in the
// Test Pilot experiment, so exclude them.
return super.isEligible();
}
whenEligible () {
}
whenInstalled () {
tabs.open(`data:text/html, Thank you for helping us study Containers in Firefox. You are in the ${this.variation} variation.`);
}
cleanup() {
}
}
const thisStudy = new ContainersStudy(studyConfig);
unload((reason) => thisStudy.shutdown(reason));
exports.study = thisStudy;

View file

@ -1,6 +1,9 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
const Experiment = require('./lib/testpilot/experiment');
const experiment = new Experiment();
/**
* Class that represents a metrics event broker. Events are sent to Google