"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.parseModelStateReasonFromStats = exports.parseModelStateFromStats = exports.parseMlInferenceParametersFromPipeline = exports.getMlModelTypesForModelConfig = exports.getMlInferencePrefixedFieldName = exports.getInferenceProcessor = exports.generateMlInferencePipelineBody = exports.formatPipelineName = exports.TEXT_EXPANSION_TYPE = exports.TEXT_EXPANSION_FRIENDLY_TYPE = exports.ML_INFERENCE_PREFIX = exports.ELSER_MODEL_ID = void 0; var _mlTrainedModelsUtils = require("@kbn/ml-trained-models-utils"); var _pipelines = require("../types/pipelines"); /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ const TEXT_EXPANSION_TYPE = _mlTrainedModelsUtils.SUPPORTED_PYTORCH_TASKS.TEXT_EXPANSION; exports.TEXT_EXPANSION_TYPE = TEXT_EXPANSION_TYPE; const TEXT_EXPANSION_FRIENDLY_TYPE = 'ELSER'; exports.TEXT_EXPANSION_FRIENDLY_TYPE = TEXT_EXPANSION_FRIENDLY_TYPE; const ML_INFERENCE_PREFIX = 'ml.inference.'; exports.ML_INFERENCE_PREFIX = ML_INFERENCE_PREFIX; const ELSER_MODEL_ID = '.elser_model_1'; exports.ELSER_MODEL_ID = ELSER_MODEL_ID; /** * Generates the pipeline body for a machine learning inference pipeline * @param pipelineConfiguration machine learning inference pipeline configuration parameters * @returns pipeline body */ const generateMlInferencePipelineBody = ({ description, fieldMappings, inferenceConfig, model, pipelineName }) => { const pipelineDefinition = { description: description !== null && description !== void 0 ? description : '', processors: [], version: 1 }; pipelineDefinition.processors = [ // Add remove and inference processors ...fieldMappings.flatMap(({ sourceField, targetField }) => { const inference = getInferenceProcessor(sourceField, targetField, inferenceConfig, model, pipelineName); return [{ remove: { field: targetField, ignore_missing: true } }, { inference }]; }), // Add single append processor { append: { field: '_source._ingest.processors', value: [{ model_version: model.version, pipeline: pipelineName, processed_timestamp: '{{{ _ingest.timestamp }}}', types: getMlModelTypesForModelConfig(model) }] } }]; return pipelineDefinition; }; exports.generateMlInferencePipelineBody = generateMlInferencePipelineBody; const getInferenceProcessor = (sourceField, targetField, inferenceConfig, model, pipelineName) => { var _model$input, _model$input$field_na; // If model returned no input field, insert a placeholder const modelInputField = ((_model$input = model.input) === null || _model$input === void 0 ? void 0 : (_model$input$field_na = _model$input.field_names) === null || _model$input$field_na === void 0 ? void 0 : _model$input$field_na.length) > 0 ? model.input.field_names[0] : 'MODEL_INPUT_FIELD'; return { field_map: { [sourceField]: modelInputField }, inference_config: inferenceConfig, model_id: model.model_id, on_failure: [{ append: { field: '_source._ingest.inference_errors', allow_duplicates: false, value: [{ message: `Processor 'inference' in pipeline '${pipelineName}' failed for field '${sourceField}' with message '{{ _ingest.on_failure_message }}'`, pipeline: pipelineName, timestamp: '{{{ _ingest.timestamp }}}' }] } }], target_field: targetField }; }; /** * Parses model types list from the given configuration of a trained machine learning model * @param trainedModel configuration for a trained machine learning model * @returns list of model types */ exports.getInferenceProcessor = getInferenceProcessor; const getMlModelTypesForModelConfig = trainedModel => { var _trainedModel$tags; if (!trainedModel) return []; const isBuiltIn = (_trainedModel$tags = trainedModel.tags) === null || _trainedModel$tags === void 0 ? void 0 : _trainedModel$tags.includes(_mlTrainedModelsUtils.BUILT_IN_MODEL_TAG); return [trainedModel.model_type, ...Object.keys(trainedModel.inference_config || {}), ...(isBuiltIn ? [_mlTrainedModelsUtils.BUILT_IN_MODEL_TAG] : [])].filter(type => type !== undefined); }; exports.getMlModelTypesForModelConfig = getMlModelTypesForModelConfig; const formatPipelineName = rawName => rawName.trim().replace(/\s+/g, '_') // Convert whitespaces to underscores .toLowerCase(); exports.formatPipelineName = formatPipelineName; const parseMlInferenceParametersFromPipeline = (name, pipeline) => { var _pipeline$processors; const inferenceProcessors = pipeline === null || pipeline === void 0 ? void 0 : (_pipeline$processors = pipeline.processors) === null || _pipeline$processors === void 0 ? void 0 : _pipeline$processors.filter(p => p.inference).map(p => p.inference); if (!inferenceProcessors || inferenceProcessors.length === 0) { return null; } // Extract source -> target field mappings from all inference processors in pipeline const fieldMappings = inferenceProcessors.map(p => { var _p$field_map; const sourceFields = Object.keys((_p$field_map = p.field_map) !== null && _p$field_map !== void 0 ? _p$field_map : {}); // We assume that there is only one source field per inference processor const sourceField = sourceFields.length >= 1 ? sourceFields[0] : null; return { sourceField, targetField: p.target_field // Prefixed target field }; }).filter(f => f.sourceField); return fieldMappings.length === 0 ? null : { model_id: inferenceProcessors[0].model_id, pipeline_name: name, pipeline_definition: {}, field_mappings: fieldMappings }; }; exports.parseMlInferenceParametersFromPipeline = parseMlInferenceParametersFromPipeline; const parseModelStateFromStats = (model, modelTypes) => { var _model$deployment_sta; if ((model === null || model === void 0 ? void 0 : model.model_type) === _mlTrainedModelsUtils.TRAINED_MODEL_TYPE.LANG_IDENT || modelTypes !== null && modelTypes !== void 0 && modelTypes.includes(_mlTrainedModelsUtils.TRAINED_MODEL_TYPE.LANG_IDENT)) return _pipelines.TrainedModelState.Started; switch (model === null || model === void 0 ? void 0 : (_model$deployment_sta = model.deployment_stats) === null || _model$deployment_sta === void 0 ? void 0 : _model$deployment_sta.state) { case 'started': return _pipelines.TrainedModelState.Started; case 'starting': return _pipelines.TrainedModelState.Starting; case 'stopping': return _pipelines.TrainedModelState.Stopping; // @ts-ignore: type is wrong, "failed" is a possible state case 'failed': return _pipelines.TrainedModelState.Failed; default: return _pipelines.TrainedModelState.NotDeployed; } }; exports.parseModelStateFromStats = parseModelStateFromStats; const parseModelStateReasonFromStats = trainedModelStats => { var _trainedModelStats$de; return trainedModelStats === null || trainedModelStats === void 0 ? void 0 : (_trainedModelStats$de = trainedModelStats.deployment_stats) === null || _trainedModelStats$de === void 0 ? void 0 : _trainedModelStats$de.reason; }; exports.parseModelStateReasonFromStats = parseModelStateReasonFromStats; const getMlInferencePrefixedFieldName = fieldName => fieldName.startsWith(ML_INFERENCE_PREFIX) ? fieldName : `${ML_INFERENCE_PREFIX}${fieldName}`; exports.getMlInferencePrefixedFieldName = getMlInferencePrefixedFieldName;