diff --git a/README.md b/README.md index e9dd52a538cd..eed1c9ca33ce 100644 --- a/README.md +++ b/README.md @@ -124,3 +124,12 @@ services. Read our learn more. This extension respects the `telemetry.telemetryLevel` setting which you can learn more about at https://code.visualstudio.com/docs/supporting/faq#_how-to-disable-telemetry-reporting. + +When usage telemetry is enabled, the existing startup event can include whether the +Python Environments extension was available in Python's extension host and whether +the resolved integration setting was enabled when Python first made its cached +integration decision. These two boolean properties (`envsAvailableAtDecision` and +`envsEnabledAtDecision`) reuse existing checks; they do not activate the extension, +fetch experiment assignments, or send additional events. They describe the first +decision, not current installation status or explicit user intent, and are omitted +when that decision has not been made or startup properties cannot be collected. diff --git a/src/client/envExt/api.internal.ts b/src/client/envExt/api.internal.ts index 5edfb712072e..f5e1e813543a 100644 --- a/src/client/envExt/api.internal.ts +++ b/src/client/envExt/api.internal.ts @@ -55,14 +55,28 @@ export function shouldEnvExtHandleActivation(): boolean { } let _useExt: boolean | undefined; +interface EnvironmentsExtensionDecisionTelemetry { + readonly envsAvailableAtDecision: boolean; + readonly envsEnabledAtDecision: boolean; +} + +let decisionTelemetry: EnvironmentsExtensionDecisionTelemetry | undefined; + +/** Reads the first decision's inputs without initializing or recomputing the cached decision. */ +export function getEnvironmentsExtensionDecisionTelemetry(): EnvironmentsExtensionDecisionTelemetry | undefined { + return decisionTelemetry; +} + export function useEnvExtension(): boolean { if (_useExt !== undefined) { return _useExt; } const config = getConfiguration('python'); const inExpSetting = config?.get('useEnvironmentsExtension', false) ?? false; + const available = !!getExtension(ENVS_EXTENSION_ID); // If extension is installed and in experiment, then use it. - _useExt = !!getExtension(ENVS_EXTENSION_ID) && inExpSetting; + _useExt = available && inExpSetting; + decisionTelemetry = { envsAvailableAtDecision: available, envsEnabledAtDecision: inExpSetting }; return _useExt; } diff --git a/src/client/startupTelemetry.ts b/src/client/startupTelemetry.ts index 40a5d6b87e7d..ff42c5325927 100644 --- a/src/client/startupTelemetry.ts +++ b/src/client/startupTelemetry.ts @@ -16,7 +16,7 @@ import { sendTelemetryEvent } from './telemetry'; import { EventName } from './telemetry/constants'; import { EditorLoadTelemetry } from './telemetry/types'; import { IStartupDurations } from './types'; -import { useEnvExtension } from './envExt/api.internal'; +import { getEnvironmentsExtensionDecisionTelemetry, useEnvExtension } from './envExt/api.internal'; import { getEnvsExplicitFalseScope } from './envExt/telemetry'; export async function sendStartupTelemetry( @@ -99,6 +99,7 @@ async function getActivationTelemetryProps( terminal: terminalShellType, isFirstSession, envsExplicitFalseScope: getEnvsExplicitFalseScope(), + ...getEnvironmentsExtensionDecisionTelemetry(), }; } const interpreterService = serviceContainer.get(IInterpreterService); @@ -157,5 +158,6 @@ async function getActivationTelemetryProps( isFirstSession, usingEnvironmentsExtension, envsExplicitFalseScope: getEnvsExplicitFalseScope(), + ...getEnvironmentsExtensionDecisionTelemetry(), }; } diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 7d95dc9b9d31..14e847bea4be 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -360,7 +360,9 @@ export interface IEventNamePropertyMapping { "usingglobalinterpreter" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" }, "isfirstsession" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" }, "usingenvironmentsextension" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eduardovil" }, - "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" } + "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsavailableatdecision" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }, + "envsenabledatdecision" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" } } */ [EventName.EDITOR_LOAD]: { @@ -416,6 +418,16 @@ export interface IEventNamePropertyMapping { * 'none' excludes defaults; 'multiple' means more than one of user, workspace, or folder. */ envsExplicitFalseScope?: EnvsExplicitFalseScope; + /** + * Whether Environments was visible in Python's extension host at the first cached integration decision. + * Omitted if no decision has been made; not an activation or current installation-status signal. + */ + envsAvailableAtDecision?: boolean; + /** + * Resolved integration setting at the first cached decision, including defaults and experiment overrides. + * Omitted if no decision has been made; not explicit user intent or a later setting snapshot. + */ + envsEnabledAtDecision?: boolean; }; /** * Reports explicit-false scopes after a python.useEnvironmentsExtension configuration change. diff --git a/src/test/envExt/decisionTelemetry.unit.test.ts b/src/test/envExt/decisionTelemetry.unit.test.ts new file mode 100644 index 000000000000..24e29dd172d0 --- /dev/null +++ b/src/test/envExt/decisionTelemetry.unit.test.ts @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { assert } from 'chai'; +import rewiremock from 'rewiremock'; +import * as sinon from 'sinon'; + +suite('Environments extension decision telemetry', () => { + let api: typeof import('../../client/envExt/api.internal'); + let getConfiguration: sinon.SinonStub; + let getSetting: sinon.SinonStub; + let getExtension: sinon.SinonStub; + let activate: sinon.SinonStub; + + setup(() => { + getSetting = sinon.stub().returns(true); + getConfiguration = sinon.stub().returns({ get: getSetting }); + activate = sinon.stub(); + getExtension = sinon.stub().returns({ activate }); + api = rewiremock.proxy( + () => require('../../client/envExt/api.internal'), + { + [require.resolve('../../client/common/vscodeApis/workspaceApis')]: { getConfiguration }, + [require.resolve('../../client/common/vscodeApis/extensionsApi')]: { getExtension }, + }, + ); + }); + + teardown(() => { + rewiremock.disable(); + sinon.restore(); + }); + + test('Reading telemetry does not initialize the decision or inspect the host', () => { + assert.isUndefined(api.getEnvironmentsExtensionDecisionTelemetry()); + assert.isUndefined(api.getEnvironmentsExtensionDecisionTelemetry()); + sinon.assert.notCalled(getConfiguration); + sinon.assert.notCalled(getSetting); + sinon.assert.notCalled(getExtension); + sinon.assert.notCalled(activate); + }); + + [false, true].forEach((available) => { + [false, true].forEach((enabled) => { + test(`Captures original inputs once: available=${available}, enabled=${enabled}`, () => { + getExtension.returns(available ? { activate } : undefined); + getSetting.returns(enabled); + + assert.strictEqual(api.useEnvExtension(), available && enabled); + assert.deepEqual(api.getEnvironmentsExtensionDecisionTelemetry(), { + envsAvailableAtDecision: available, + envsEnabledAtDecision: enabled, + }); + + getExtension.returns(available ? undefined : { activate }); + getSetting.returns(!enabled); + assert.strictEqual(api.useEnvExtension(), available && enabled); + assert.deepEqual(api.getEnvironmentsExtensionDecisionTelemetry(), { + envsAvailableAtDecision: available, + envsEnabledAtDecision: enabled, + }); + sinon.assert.calledOnceWithExactly(getConfiguration, 'python'); + sinon.assert.calledOnceWithExactly(getSetting, 'useEnvironmentsExtension', false); + sinon.assert.calledOnceWithExactly(getExtension, api.ENVS_EXTENSION_ID); + sinon.assert.callOrder(getConfiguration, getSetting, getExtension); + sinon.assert.notCalled(activate); + }); + }); + }); + + test('Preserves the false fallback for an unavailable setting value', () => { + getSetting.returns(undefined); + assert.isFalse(api.useEnvExtension()); + assert.deepEqual(api.getEnvironmentsExtensionDecisionTelemetry(), { + envsAvailableAtDecision: true, + envsEnabledAtDecision: false, + }); + }); + + test('Preserves the false fallback when configuration is unavailable', () => { + getConfiguration.returns(undefined); + assert.isFalse(api.useEnvExtension()); + assert.deepEqual(api.getEnvironmentsExtensionDecisionTelemetry(), { + envsAvailableAtDecision: true, + envsEnabledAtDecision: false, + }); + sinon.assert.notCalled(getSetting); + sinon.assert.calledOnce(getExtension); + }); + + test('Failed configuration lookup preserves lookup order and leaves inputs unknown', () => { + getSetting.throws(new Error('configuration failed')); + assert.throws(() => api.useEnvExtension(), 'configuration failed'); + assert.isUndefined(api.getEnvironmentsExtensionDecisionTelemetry()); + sinon.assert.notCalled(getExtension); + }); + + test('Failed lookup does not fabricate a decision snapshot', () => { + getExtension.throws(new Error('lookup failed')); + assert.throws(() => api.useEnvExtension(), 'lookup failed'); + assert.isUndefined(api.getEnvironmentsExtensionDecisionTelemetry()); + getExtension.returns({ activate }); + assert.isTrue(api.useEnvExtension()); + assert.deepEqual(api.getEnvironmentsExtensionDecisionTelemetry(), { + envsAvailableAtDecision: true, + envsEnabledAtDecision: true, + }); + }); +}); diff --git a/src/test/envExt/startupDecisionTelemetry.unit.test.ts b/src/test/envExt/startupDecisionTelemetry.unit.test.ts new file mode 100644 index 000000000000..c5ef5775b127 --- /dev/null +++ b/src/test/envExt/startupDecisionTelemetry.unit.test.ts @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { assert } from 'chai'; +import * as sinon from 'sinon'; +import * as TypeMoq from 'typemoq'; +import { IWorkspaceService } from '../../client/common/application/types'; +import * as constants from '../../client/common/constants'; +import { ITerminalHelper, TerminalShellType } from '../../client/common/terminal/types'; +import { IInterpreterPathService } from '../../client/common/types'; +import * as envExt from '../../client/envExt/api.internal'; +import * as envExtTelemetry from '../../client/envExt/telemetry'; +import { IInterpreterService } from '../../client/interpreter/contracts'; +import { IServiceContainer } from '../../client/ioc/types'; +import { sendErrorTelemetry, sendStartupTelemetry } from '../../client/startupTelemetry'; +import * as telemetry from '../../client/telemetry'; +import { EventName } from '../../client/telemetry/constants'; +import { IStartupDurations } from '../../client/types'; + +suite('Startup Telemetry - Environments decision inputs', () => { + let container: TypeMoq.IMock; + let workspace: TypeMoq.IMock; + let useEnvExtension: sinon.SinonStub; + let getDecision: sinon.SinonStub; + let send: sinon.SinonStub; + let durations: IStartupDurations; + + setup(() => { + container = TypeMoq.Mock.ofType(); + workspace = TypeMoq.Mock.ofType(); + const terminal = TypeMoq.Mock.ofType(); + terminal.setup((t) => t.identifyTerminalShell()).returns(() => TerminalShellType.bash); + const interpreter = TypeMoq.Mock.ofType(); + interpreter.setup((i) => i.hasInterpreters(TypeMoq.It.isAny())).returns(async () => false); + interpreter.setup((i) => i.refreshPromise).returns(() => Promise.resolve()); + interpreter.setup((i) => i.getActiveInterpreter()).returns(async () => undefined); + interpreter.setup((i) => i.getActiveInterpreter(TypeMoq.It.isAny())).returns(async () => undefined); + const paths = TypeMoq.Mock.ofType(); + paths + .setup((p) => p.inspect(TypeMoq.It.isAny())) + .returns(() => ({ + globalValue: undefined, + workspaceValue: undefined, + workspaceFolderValue: undefined, + })); + container.setup((c) => c.get(IWorkspaceService)).returns(() => workspace.object); + container.setup((c) => c.get(ITerminalHelper)).returns(() => terminal.object); + container.setup((c) => c.get(IInterpreterService)).returns(() => interpreter.object); + container.setup((c) => c.get(IInterpreterPathService)).returns(() => paths.object); + sinon.stub(constants, 'isTestExecution').returns(false); + useEnvExtension = sinon.stub(envExt, 'useEnvExtension').returns(false); + getDecision = sinon.stub(envExt, 'getEnvironmentsExtensionDecisionTelemetry'); + sinon.stub(envExtTelemetry, 'getEnvsExplicitFalseScope').returns('none'); + send = sinon.stub(telemetry, 'sendTelemetryEvent'); + durations = { + startActivateTime: 0, + totalActivateTime: 0, + totalNonBlockingActivateTime: 0, + codeLoadingTime: 0, + }; + }); + + teardown(() => sinon.restore()); + + [false, true].forEach((available) => { + [false, true].forEach((enabled) => { + test(`Trusted startup emits decision inputs: available=${available}, enabled=${enabled}`, async () => { + workspace.setup((w) => w.isTrusted).returns(() => true); + useEnvExtension.returns(available && enabled); + const snapshot = { envsAvailableAtDecision: available, envsEnabledAtDecision: enabled }; + getDecision.returns(snapshot); + + await sendStartupTelemetry(Promise.resolve(), durations, { elapsedTime: 10 }, container.object, false); + + sinon.assert.calledOnceWithExactly( + send, + EventName.EDITOR_LOAD, + durations, + sinon.match({ + ...snapshot, + usingEnvironmentsExtension: available && enabled, + envsExplicitFalseScope: 'none', + }), + ); + sinon.assert.calledTwice(useEnvExtension); + sinon.assert.calledOnce(getDecision); + sinon.assert.callOrder(useEnvExtension, getDecision, send); + }); + }); + }); + + [false, true].forEach((hasDecision) => { + test(`Untrusted startup reads only existing inputs: hasDecision=${hasDecision}`, async () => { + workspace.setup((w) => w.isTrusted).returns(() => false); + const snapshot = { envsAvailableAtDecision: false, envsEnabledAtDecision: true }; + getDecision.returns(hasDecision ? snapshot : undefined); + + await sendStartupTelemetry(Promise.resolve(), durations, { elapsedTime: 10 }, container.object, false); + + sinon.assert.notCalled(useEnvExtension); + sinon.assert.calledOnce(getDecision); + sinon.assert.calledOnceWithExactly(send, EventName.EDITOR_LOAD, durations, { + workspaceFolderCount: 0, + terminal: TerminalShellType.bash, + isFirstSession: false, + envsExplicitFalseScope: 'none', + ...(hasDecision ? snapshot : {}), + }); + }); + }); + + test('Error-only startup without services leaves inputs absent and does not compute a decision', async () => { + const error = new Error('activation failed'); + await sendErrorTelemetry(error, durations); + sinon.assert.calledOnceWithExactly(send, EventName.EDITOR_LOAD, durations, {}, error); + sinon.assert.notCalled(getDecision); + sinon.assert.notCalled(useEnvExtension); + }); + + test('Error telemetry with services includes already captured inputs', async () => { + workspace.setup((w) => w.isTrusted).returns(() => false); + getDecision.returns({ envsAvailableAtDecision: true, envsEnabledAtDecision: false }); + const error = new Error('activation failed'); + await sendErrorTelemetry(error, durations, container.object); + sinon.assert.calledOnceWithExactly( + send, + EventName.EDITOR_LOAD, + durations, + sinon.match({ envsAvailableAtDecision: true, envsEnabledAtDecision: false }), + error, + ); + sinon.assert.notCalled(useEnvExtension); + }); + + test('Existing test-execution guard still suppresses startup collection', async () => { + sinon.restore(); + sinon.stub(constants, 'isTestExecution').returns(true); + const decision = sinon.spy(envExt, 'getEnvironmentsExtensionDecisionTelemetry'); + const sender = sinon.spy(telemetry, 'sendTelemetryEvent'); + await sendStartupTelemetry(Promise.resolve(), durations, { elapsedTime: 10 }, container.object, false); + assert.isFalse(decision.called); + assert.isFalse(sender.called); + }); +}); diff --git a/src/test/telemetry/index.unit.test.ts b/src/test/telemetry/index.unit.test.ts index 774270308ca1..48cbaada51ce 100644 --- a/src/test/telemetry/index.unit.test.ts +++ b/src/test/telemetry/index.unit.test.ts @@ -6,6 +6,8 @@ import { expect } from 'chai'; import rewiremock from 'rewiremock'; import * as sinon from 'sinon'; import * as fs from '../../client/common/platform/fs-paths'; +import { TerminalShellType } from '../../client/common/terminal/types'; +import { EventName } from '../../client/telemetry/constants'; import { _resetSharedProperties, @@ -85,6 +87,38 @@ suite('Telemetry', () => { expect(Reporter.measures).to.deep.equal([undefined], 'Measures should be empty'); expect(Reporter.properties).to.deep.equal([{}], 'Properties should be empty'); }); + test('Serialize Environments decision booleans on the existing startup event', () => { + rewiremock.enable(); + rewiremock('@vscode/extension-telemetry').with({ TelemetryReporter: Reporter }); + + sendTelemetryEvent(EventName.EDITOR_LOAD, undefined, { + terminal: TerminalShellType.bash, + workspaceFolderCount: 0, + envsAvailableAtDecision: false, + envsEnabledAtDecision: true, + }); + + expect(Reporter.eventName).to.deep.equal([EventName.EDITOR_LOAD]); + expect(Reporter.properties[0]).to.include({ + envsAvailableAtDecision: 'false', + envsEnabledAtDecision: 'true', + }); + }); + test('Unknown Environments decision inputs are omitted from the startup payload', () => { + rewiremock.enable(); + rewiremock('@vscode/extension-telemetry').with({ TelemetryReporter: Reporter }); + + sendTelemetryEvent(EventName.EDITOR_LOAD, undefined, { + terminal: TerminalShellType.bash, + workspaceFolderCount: 0, + envsAvailableAtDecision: undefined, + envsEnabledAtDecision: undefined, + }); + + expect(Reporter.eventName).to.deep.equal([EventName.EDITOR_LOAD]); + expect(Reporter.properties[0]).not.to.have.property('envsAvailableAtDecision'); + expect(Reporter.properties[0]).not.to.have.property('envsEnabledAtDecision'); + }); test('Send Telemetry with shared properties', () => { rewiremock.enable(); rewiremock('@vscode/extension-telemetry').with({ TelemetryReporter: Reporter });