"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports._deleteSecrets = _deleteSecrets; exports.createSecrets = createSecrets; exports.deleteSecretsIfNotReferenced = deleteSecretsIfNotReferenced; exports.diffSecretPaths = diffSecretPaths; exports.extractAndUpdateSecrets = extractAndUpdateSecrets; exports.extractAndWriteSecrets = extractAndWriteSecrets; exports.findPackagePoliciesUsingSecrets = findPackagePoliciesUsingSecrets; exports.getPolicySecretPaths = getPolicySecretPaths; exports.isSecretStorageEnabled = isSecretStorageEnabled; exports.toCompiledSecretRef = toCompiledSecretRef; var _lodash = require("lodash"); var _saferLodashSet = require("@kbn/safer-lodash-set"); var _policy_template = require("../../common/services/policy_template"); var _common = require("../../common"); var _services = require("../../common/services"); var _errors = require("../errors"); var _constants = require("../constants"); var _retry = require("./epm/elasticsearch/retry"); var _audit_logging = require("./audit_logging"); var _app_context = require("./app_context"); var _package_policy = require("./package_policy"); var _ = require("."); var _fleet_server = require("./fleet_server"); /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ async function createSecrets(opts) { const { esClient, values } = opts; const logger = _app_context.appContextService.getLogger(); const secretsResponse = await Promise.all(values.map(async value => { try { return await (0, _retry.retryTransientEsErrors)(() => esClient.transport.request({ method: 'POST', path: _constants.SECRETS_ENDPOINT_PATH, body: { value } }), { logger }); } catch (err) { const msg = `Error creating secrets: ${err}`; logger.error(msg); throw new _errors.FleetError(msg); } })); secretsResponse.forEach(item => { _audit_logging.auditLoggingService.writeCustomAuditLog({ message: `secret created: ${item.id}`, event: { action: 'secret_create', category: ['database'], type: ['access'], outcome: 'success' } }); }); return secretsResponse; } async function deleteSecretsIfNotReferenced(opts) { const { esClient, soClient, ids } = opts; const logger = _app_context.appContextService.getLogger(); const packagePoliciesUsingSecrets = await findPackagePoliciesUsingSecrets({ soClient, ids }); if (packagePoliciesUsingSecrets.length) { packagePoliciesUsingSecrets.forEach(({ id, policyIds }) => { logger.debug(`Not deleting secret with id ${id} is still in use by package policies: ${policyIds.join(', ')}`); }); } const secretsToDelete = ids.filter(id => { return !packagePoliciesUsingSecrets.some(packagePolicy => packagePolicy.id === id); }); if (!secretsToDelete.length) { return; } try { await _deleteSecrets({ esClient, ids: secretsToDelete }); } catch (e) { logger.warn(`Error cleaning up secrets ${ids.join(', ')}: ${e}`); } } async function findPackagePoliciesUsingSecrets(opts) { const { soClient, ids } = opts; const packagePolicies = await _package_policy.packagePolicyService.list(soClient, { kuery: `ingest-package-policies.secret_references.id: (${ids.join(' or ')})`, perPage: _common.SO_SEARCH_LIMIT, page: 1 }); if (!packagePolicies.total) { return []; } // create a map of secret_references.id to package policy id const packagePoliciesBySecretId = packagePolicies.items.reduce((acc, packagePolicy) => { var _packagePolicy$secret; packagePolicy === null || packagePolicy === void 0 ? void 0 : (_packagePolicy$secret = packagePolicy.secret_references) === null || _packagePolicy$secret === void 0 ? void 0 : _packagePolicy$secret.forEach(secretReference => { if (!acc[secretReference.id]) { acc[secretReference.id] = []; } acc[secretReference.id].push(packagePolicy.id); }); return acc; }, {}); const res = []; for (const id of ids) { if (packagePoliciesBySecretId[id]) { res.push({ id, policyIds: packagePoliciesBySecretId[id] }); } } return res; } async function _deleteSecrets(opts) { const { esClient, ids } = opts; const logger = _app_context.appContextService.getLogger(); const deletedRes = await Promise.all(ids.map(async id => { try { const getDeleteRes = await (0, _retry.retryTransientEsErrors)(() => esClient.transport.request({ method: 'DELETE', path: `${_constants.SECRETS_ENDPOINT_PATH}/${id}` }), { logger }); return { ...getDeleteRes, id }; } catch (err) { const msg = `Error deleting secrets: ${err}`; logger.error(msg); throw new _errors.FleetError(msg); } })); deletedRes.forEach(item => { if (item.deleted === true) { _audit_logging.auditLoggingService.writeCustomAuditLog({ message: `secret deleted: ${item.id}`, event: { action: 'secret_delete', category: ['database'], type: ['access'], outcome: 'success' } }); } }); } async function extractAndWriteSecrets(opts) { const { packagePolicy, packageInfo, esClient } = opts; const secretPaths = getPolicySecretPaths(packagePolicy, packageInfo); if (!secretPaths.length) { return { packagePolicy, secretReferences: [] }; } const secrets = await createSecrets({ esClient, values: secretPaths.map(secretPath => secretPath.value.value) }); const policyWithSecretRefs = JSON.parse(JSON.stringify(packagePolicy)); secretPaths.forEach((secretPath, i) => { (0, _saferLodashSet.set)(policyWithSecretRefs, secretPath.path + '.value', toVarSecretRef(secrets[i].id)); }); return { packagePolicy: policyWithSecretRefs, secretReferences: secrets.map(({ id }) => ({ id })) }; } async function extractAndUpdateSecrets(opts) { const { oldPackagePolicy, packagePolicyUpdate, packageInfo, esClient } = opts; const oldSecretPaths = getPolicySecretPaths(oldPackagePolicy, packageInfo); const updatedSecretPaths = getPolicySecretPaths(packagePolicyUpdate, packageInfo); if (!oldSecretPaths.length && !updatedSecretPaths.length) { return { packagePolicyUpdate, secretReferences: [], secretsToDelete: [] }; } const { toCreate, toDelete, noChange } = diffSecretPaths(oldSecretPaths, updatedSecretPaths); const createdSecrets = await createSecrets({ esClient, values: toCreate.map(secretPath => secretPath.value.value) }); const policyWithSecretRefs = JSON.parse(JSON.stringify(packagePolicyUpdate)); toCreate.forEach((secretPath, i) => { (0, _saferLodashSet.set)(policyWithSecretRefs, secretPath.path + '.value', toVarSecretRef(createdSecrets[i].id)); }); const secretReferences = [...noChange.map(secretPath => ({ id: secretPath.value.value.id })), ...createdSecrets.map(({ id }) => ({ id }))]; const secretsToDelete = []; toDelete.forEach(secretPath => { // check if the previous secret is actually a secret refrerence // it may be that secrets were not enabled at the time of creation // in which case they are just stored as plain text if (secretPath.value.value.isSecretRef) { secretsToDelete.push({ id: secretPath.value.value.id }); } }); return { packagePolicyUpdate: policyWithSecretRefs, secretReferences, secretsToDelete }; } function isSecretVar(varDef) { return varDef.secret === true; } function containsSecretVar(vars) { return vars === null || vars === void 0 ? void 0 : vars.some(isSecretVar); } // this is how secrets are stored on the package policy function toVarSecretRef(id) { return { id, isSecretRef: true }; } // this is how IDs are inserted into compiled templates function toCompiledSecretRef(id) { return `$co.elastic.secret{${id}}`; } function diffSecretPaths(oldPaths, newPaths) { const toCreate = []; const toDelete = []; const noChange = []; const newPathsByPath = (0, _lodash.keyBy)(newPaths, 'path'); for (const oldPath of oldPaths) { if (!newPathsByPath[oldPath.path]) { toDelete.push(oldPath); } const newPath = newPathsByPath[oldPath.path]; if (newPath && newPath.value.value) { var _newPath$value; const newValue = (_newPath$value = newPath.value) === null || _newPath$value === void 0 ? void 0 : _newPath$value.value; if (!(newValue !== null && newValue !== void 0 && newValue.isSecretRef)) { toCreate.push(newPath); toDelete.push(oldPath); } else { noChange.push(newPath); } delete newPathsByPath[oldPath.path]; } } const remainingNewPaths = Object.values(newPathsByPath); return { toCreate: [...toCreate, ...remainingNewPaths], toDelete, noChange }; } // Given a package policy and a package, // returns an array of lodash style paths to all secrets and their current values function getPolicySecretPaths(packagePolicy, packageInfo) { var _packageInfo$policy_t; const packageLevelVarPaths = _getPackageLevelSecretPaths(packagePolicy, packageInfo); if (!(packageInfo !== null && packageInfo !== void 0 && (_packageInfo$policy_t = packageInfo.policy_templates) !== null && _packageInfo$policy_t !== void 0 && _packageInfo$policy_t.length) || (0, _policy_template.packageHasNoPolicyTemplates)(packageInfo)) { return packageLevelVarPaths; } const inputSecretPaths = _getInputSecretPaths(packagePolicy, packageInfo); return [...packageLevelVarPaths, ...inputSecretPaths]; } async function isSecretStorageEnabled(esClient, soClient) { var _appContextService$ge, _appContextService$ge2, _appContextService$ge3; const logger = _app_context.appContextService.getLogger(); // first check if the feature flag is enabled, if not secrets are disabled const { secretsStorage: secretsStorageEnabled } = _app_context.appContextService.getExperimentalFeatures(); if (!secretsStorageEnabled) { logger.debug('Secrets storage is disabled by feature flag'); return false; } // if serverless then secrets will always be supported const isFleetServerStandalone = (_appContextService$ge = (_appContextService$ge2 = _app_context.appContextService.getConfig()) === null || _appContextService$ge2 === void 0 ? void 0 : (_appContextService$ge3 = _appContextService$ge2.internal) === null || _appContextService$ge3 === void 0 ? void 0 : _appContextService$ge3.fleetServerStandalone) !== null && _appContextService$ge !== void 0 ? _appContextService$ge : false; if (isFleetServerStandalone) { logger.trace('Secrets storage is enabled as fleet server is standalone'); return true; } // now check the flag in settings to see if the fleet server requirement has already been met // once the requirement has been met, secrets are always on const settings = await _.settingsService.getSettings(soClient); if (settings.secret_storage_requirements_met) { logger.debug('Secrets storage already met, turned on is settings'); return true; } // otherwise check if we have the minimum fleet server version and enable secrets if so if (await (0, _fleet_server.allFleetServerVersionsAreAtLeast)(esClient, soClient, _constants.SECRETS_MINIMUM_FLEET_SERVER_VERSION)) { logger.debug('Enabling secrets storage as minimum fleet server version has been met'); try { await _.settingsService.saveSettings(soClient, { secret_storage_requirements_met: true }); } catch (err) { // we can suppress this error as it will be retried on the next function call logger.warn(`Failed to save settings after enabling secrets storage: ${err.message}`); } return true; } logger.info('Secrets storage is disabled as minimum fleet server version has not been met'); return false; } function _getPackageLevelSecretPaths(packagePolicy, packageInfo) { var _packageInfo$vars; const packageSecretVars = ((_packageInfo$vars = packageInfo.vars) === null || _packageInfo$vars === void 0 ? void 0 : _packageInfo$vars.filter(isSecretVar)) || []; const packageSecretVarsByName = (0, _lodash.keyBy)(packageSecretVars, 'name'); const packageVars = Object.entries(packagePolicy.vars || {}); return packageVars.reduce((vars, [name, configEntry], i) => { if (packageSecretVarsByName[name]) { vars.push({ value: configEntry, path: `vars.${name}` }); } return vars; }, []); } function _getInputSecretPaths(packagePolicy, packageInfo) { var _packageInfo$policy_t2; if (!(packageInfo !== null && packageInfo !== void 0 && (_packageInfo$policy_t2 = packageInfo.policy_templates) !== null && _packageInfo$policy_t2 !== void 0 && _packageInfo$policy_t2.length)) return []; const inputSecretVarDefsByPolicyTemplateAndType = _getInputSecretVarDefsByPolicyTemplateAndType(packageInfo); const streamSecretVarDefsByDatasetAndInput = _getStreamSecretVarDefsByDatasetAndInput(packageInfo); return packagePolicy.inputs.flatMap((input, inputIndex) => { if (!input.vars && !input.streams) { return []; } const currentInputVarPaths = []; const inputKey = (0, _services.doesPackageHaveIntegrations)(packageInfo) ? `${input.policy_template}-${input.type}` : input.type; const inputVars = Object.entries(input.vars || {}); if (inputVars.length) { inputVars.forEach(([name, configEntry]) => { var _inputSecretVarDefsBy; if ((_inputSecretVarDefsBy = inputSecretVarDefsByPolicyTemplateAndType[inputKey]) !== null && _inputSecretVarDefsBy !== void 0 && _inputSecretVarDefsBy[name]) { currentInputVarPaths.push({ path: `inputs[${inputIndex}].vars.${name}`, value: configEntry }); } }); } if (input.streams.length) { input.streams.forEach((stream, streamIndex) => { const streamVarDefs = streamSecretVarDefsByDatasetAndInput[`${stream.data_stream.dataset}-${input.type}`]; if (streamVarDefs && Object.keys(streamVarDefs).length) { Object.entries(stream.vars || {}).forEach(([name, configEntry]) => { if (streamVarDefs[name]) { currentInputVarPaths.push({ path: `inputs[${inputIndex}].streams[${streamIndex}].vars.${name}`, value: configEntry }); } }); } }); } return currentInputVarPaths; }); } // a map of all secret vars for each dataset and input combo function _getStreamSecretVarDefsByDatasetAndInput(packageInfo) { const dataStreams = (0, _services.getNormalizedDataStreams)(packageInfo); const streamsByDatasetAndInput = dataStreams.reduce((streams, dataStream) => { var _dataStream$streams; (_dataStream$streams = dataStream.streams) === null || _dataStream$streams === void 0 ? void 0 : _dataStream$streams.forEach(stream => { streams[`${dataStream.dataset}-${stream.input}`] = stream; }); return streams; }, {}); return Object.entries(streamsByDatasetAndInput).reduce((varDefs, [path, stream]) => { if (stream.vars && containsSecretVar(stream.vars)) { const secretVars = stream.vars.filter(isSecretVar); varDefs[path] = (0, _lodash.keyBy)(secretVars, 'name'); } return varDefs; }, {}); } // a map of all secret vars for each policyTemplate and input type combo function _getInputSecretVarDefsByPolicyTemplateAndType(packageInfo) { var _packageInfo$policy_t3; if (!(packageInfo !== null && packageInfo !== void 0 && (_packageInfo$policy_t3 = packageInfo.policy_templates) !== null && _packageInfo$policy_t3 !== void 0 && _packageInfo$policy_t3.length)) return {}; const hasIntegrations = (0, _services.doesPackageHaveIntegrations)(packageInfo); return packageInfo.policy_templates.reduce((varDefs, policyTemplate) => { const inputs = (0, _services.getNormalizedInputs)(policyTemplate); inputs.forEach(input => { var _input$vars; const varDefKey = hasIntegrations ? `${policyTemplate.name}-${input.type}` : input.type; const secretVars = input === null || input === void 0 ? void 0 : (_input$vars = input.vars) === null || _input$vars === void 0 ? void 0 : _input$vars.filter(isSecretVar); if (secretVars !== null && secretVars !== void 0 && secretVars.length) { varDefs[varDefKey] = (0, _lodash.keyBy)(secretVars, 'name'); } }); return varDefs; }, {}); }