"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.listArtifacts = exports.getArtifact = exports.generateArtifactContentHash = exports.encodeArtifactContent = exports.deleteArtifact = exports.createArtifact = exports.bulkDeleteArtifacts = exports.bulkCreateArtifacts = exports.BULK_CREATE_MAX_ARTIFACTS_BYTES = void 0; var _zlib = require("zlib"); var _util = require("util"); var _crypto = require("crypto"); var _lodash = require("lodash"); var _common = require("../../../common"); var _errors = require("../../errors"); var _utils = require("../../errors/utils"); var _utils2 = require("../epm/packages/utils"); var _app_context = require("../app_context"); var _utils3 = require("./utils"); var _mappings = require("./mappings"); /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ const deflateAsync = (0, _util.promisify)(_zlib.deflate); const getArtifact = async (esClient, id) => { try { const esData = await esClient.get({ index: _common.FLEET_SERVER_ARTIFACTS_INDEX, id }); // @ts-expect-error @elastic/elasticsearch _source is optional return (0, _mappings.esSearchHitToArtifact)(esData); } catch (e) { if ((0, _utils3.isElasticsearchItemNotFoundError)(e)) { return; } throw new _errors.ArtifactsElasticsearchError(e); } }; exports.getArtifact = getArtifact; const createArtifact = async (esClient, artifact) => { const id = (0, _mappings.uniqueIdFromArtifact)(artifact); const newArtifactData = (0, _mappings.newArtifactToElasticsearchProperties)(artifact); try { await esClient.create({ index: _common.FLEET_SERVER_ARTIFACTS_INDEX, id, body: newArtifactData, refresh: 'wait_for' }); } catch (e) { // we ignore 409 errors from the create (document already exists) if (!(0, _utils.isElasticsearchVersionConflictError)(e)) { throw new _errors.ArtifactsElasticsearchError(e); } } return (0, _mappings.esSearchHitToArtifact)({ _id: id, _source: newArtifactData }); }; // Max length in bytes for artifacts batch exports.createArtifact = createArtifact; const BULK_CREATE_MAX_ARTIFACTS_BYTES = 4_000_000; // Function to split artifacts in batches depending on the encoded_size value. exports.BULK_CREATE_MAX_ARTIFACTS_BYTES = BULK_CREATE_MAX_ARTIFACTS_BYTES; const generateArtifactBatches = (artifacts, maxArtifactsBatchSizeInBytes = BULK_CREATE_MAX_ARTIFACTS_BYTES) => { const batches = []; let artifactsBatchLengthInBytes = 0; const sortedArtifacts = (0, _lodash.sortBy)(artifacts, 'encodedSize'); sortedArtifacts.forEach(artifact => { const esArtifact = (0, _mappings.newArtifactToElasticsearchProperties)(artifact); const bulkOperation = { create: { _id: (0, _mappings.uniqueIdFromArtifact)(artifact) } }; // Before adding the next artifact to the current batch, check if it can be added depending on the batch size limit. // If there is no artifact yet added to the current batch, we add it anyway ignoring the batch limit as the batch size has to be > 0. if (artifact.encodedSize + artifactsBatchLengthInBytes >= maxArtifactsBatchSizeInBytes) { artifactsBatchLengthInBytes = artifact.encodedSize; batches.push([bulkOperation, esArtifact]); } else { // Case it's the first one if ((0, _lodash.isEmpty)(batches)) { batches.push([]); } // Adds the next artifact to the current batch and increases the batch size count with the artifact size. artifactsBatchLengthInBytes += artifact.encodedSize; batches[batches.length - 1].push(bulkOperation, esArtifact); } }); return batches; }; const bulkCreateArtifacts = async (esClient, artifacts, refresh = false) => { var _appContextService$ge; const batches = generateArtifactBatches(artifacts, (_appContextService$ge = _app_context.appContextService.getConfig()) === null || _appContextService$ge === void 0 ? void 0 : _appContextService$ge.createArtifactsBulkBatchSize); const logger = _app_context.appContextService.getLogger(); const nonConflictErrors = []; logger.debug(`Number of batches generated for fleet artifacts: ${batches.length}`); for (let batchN = 0; batchN < batches.length; batchN++) { logger.debug(`Creating artifacts for batch ${batchN + 1} with ${batches[batchN].length / 2} artifacts`); logger.debug(`Artifacts in current batch: ${JSON.stringify(batches[batchN])}`); // Generate a bulk create for the current batch of artifacts const res = await (0, _utils2.withPackageSpan)(`Bulk create fleet artifacts batch [${batchN}]`, () => esClient.bulk({ index: _common.FLEET_SERVER_ARTIFACTS_INDEX, body: batches[batchN], refresh })); // Track errors of the bulk create action if (res.errors) { nonConflictErrors.push(...res.items.reduce((acc, item) => { var _item$create; if (((_item$create = item.create) === null || _item$create === void 0 ? void 0 : _item$create.status) !== 409) { var _item$create2, _item$create2$error; acc.push(new Error((_item$create2 = item.create) === null || _item$create2 === void 0 ? void 0 : (_item$create2$error = _item$create2.error) === null || _item$create2$error === void 0 ? void 0 : _item$create2$error.reason)); } return acc; }, [])); } } // If any non conflict error, it returns only the errors if (nonConflictErrors.length > 0) { return { errors: nonConflictErrors }; } // Use non sorted artifacts array to preserve the artifacts order in the response const nonSortedEsArtifactsResponse = artifacts.map(artifact => { return (0, _mappings.esSearchHitToArtifact)({ _id: (0, _mappings.uniqueIdFromArtifact)(artifact), _source: (0, _mappings.newArtifactToElasticsearchProperties)(artifact) }); }); return { artifacts: nonSortedEsArtifactsResponse }; }; exports.bulkCreateArtifacts = bulkCreateArtifacts; const deleteArtifact = async (esClient, id) => { try { await esClient.delete({ index: _common.FLEET_SERVER_ARTIFACTS_INDEX, id, refresh: 'wait_for' }); } catch (e) { throw new _errors.ArtifactsElasticsearchError(e); } }; exports.deleteArtifact = deleteArtifact; const bulkDeleteArtifacts = async (esClient, ids) => { try { const body = ids.map(id => ({ delete: { _index: _common.FLEET_SERVER_ARTIFACTS_INDEX, _id: id } })); const res = await (0, _utils2.withPackageSpan)(`Bulk delete fleet artifacts`, () => esClient.bulk({ body, refresh: 'wait_for' })); let errors = []; // Track errors of the bulk delete action if (res.errors) { errors = res.items.reduce((acc, item) => { var _item$delete; if ((_item$delete = item.delete) !== null && _item$delete !== void 0 && _item$delete.error) { acc.push(new Error(item.delete.error.reason)); } return acc; }, []); } return errors; } catch (e) { throw new _errors.ArtifactsElasticsearchError(e); } }; exports.bulkDeleteArtifacts = bulkDeleteArtifacts; const listArtifacts = async (esClient, options = {}) => { const { perPage = 20, page = 1, kuery = '', sortField = 'created', sortOrder = 'asc' } = options; try { const searchResult = await esClient.search({ index: _common.FLEET_SERVER_ARTIFACTS_INDEX, q: kuery, from: (page - 1) * perPage, ignore_unavailable: true, size: perPage, track_total_hits: true, rest_total_hits_as_int: true, body: { sort: [{ [sortField]: { order: sortOrder } }] } }); return { // @ts-expect-error @elastic/elasticsearch _source is optional items: searchResult.hits.hits.map(hit => (0, _mappings.esSearchHitToArtifact)(hit)), page, perPage, total: searchResult.hits.total }; } catch (e) { throw new _errors.ArtifactsElasticsearchError(e); } }; exports.listArtifacts = listArtifacts; const generateArtifactContentHash = content => { return (0, _crypto.createHash)('sha256').update(content).digest('hex'); }; exports.generateArtifactContentHash = generateArtifactContentHash; const encodeArtifactContent = async content => { const decodedContentBuffer = Buffer.from(content); const encodedContentBuffer = await deflateAsync(decodedContentBuffer); const encodedArtifact = { compressionAlgorithm: 'zlib', decodedSha256: generateArtifactContentHash(decodedContentBuffer.toString()), decodedSize: decodedContentBuffer.byteLength, encodedSha256: generateArtifactContentHash(encodedContentBuffer), encodedSize: encodedContentBuffer.byteLength, body: encodedContentBuffer.toString('base64') }; return encodedArtifact; }; exports.encodeArtifactContent = encodeArtifactContent;