Dataset import
POST
/v2/datasets/import
const url = 'http://localhost:3000/v2/datasets/import?overwrite=false';const options = { method: 'POST', headers: {'Content-Type': 'application/json'}, body: '{"id":"api.datasets.import","ver":"v2","ts":"2024-05-21T14:30:00Z","params":{"msgid":"127384e4a-a051-4a9f-9b3f-a64a8034fad7"},"request":{"id":"sb-telemetry_draft_data","dataset_id":"sb-telemetryRPF_draft_data","name":"sb-telemetry_draft_data","type":"event","extraction_config":{"is_batch_event":false,"dedup_config":{"drop_duplicates":false,"dedup_key":"id","dedup_period":604800},"extraction_key":"events"},"validation_config":{"validate":true,"mode":"Strict"},"dedup_config":{"dedup_key":"id","drop_duplicates":true,"dedup_period":604800},"data_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"eid":{"type":"string","arrival_format":"text","data_type":"string"},"ver":{"type":"string","arrival_format":"text","data_type":"string"},"syncts":{"type":"integer","arrival_format":"number","data_type":"integer"},"ets":{"type":"integer","arrival_format":"number","data_type":"epoch","suggestions":[{"message":"The Property \'ets\' appears to be \'date-time\' format type.","advice":"The System can index all data on this column","resolutionType":"INDEX","severity":"LOW","path":"properties.ets"}]},"flags":{"type":"object","properties":{"ex_processed":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"pp_validation_processed":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"pp_duplicate_skipped":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"user_denorm":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"device_denorm":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"loc_denorm":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"content_denorm":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"coll_denorm":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"mid":{"type":"string","suggestions":[{"message":"The Property \'mid\' appears to be \'uuid\' format type.","advice":"Suggest to not to index the high cardinal columns","resolutionType":"DEDUP","severity":"LOW","path":"properties.mid"}],"arrival_format":"text","data_type":"string"},"actor":{"type":"object","properties":{"id":{"type":"string","suggestions":[{"message":"The Property \'actor.id\' appears to be \'uuid\' format type.","advice":"Suggest to not to index the high cardinal columns","resolutionType":"DEDUP","severity":"LOW","path":"properties.actor.properties.id"}],"arrival_format":"text","data_type":"string"},"type":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"edata":{"type":"object","properties":{"visits":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"type":{"type":"string","arrival_format":"text","data_type":"string"},"duration":{"type":"number","arrival_format":"number","data_type":"number"},"size":{"type":"integer","arrival_format":"number","data_type":"integer"},"query":{"type":"string","arrival_format":"text","data_type":"string"},"filters":{"type":"object","properties":{"objectType":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"version":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"status":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"id":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"isRootOrg":{"type":"boolean","arrival_format":"boolean","data_type":"boolean"},"trackable.enabled":{"type":"string","arrival_format":"text","data_type":"string"},"channel":{"type":"object","arrival_format":"object","data_type":"object","additionalProperties":false},"framework":{"type":"object","arrival_format":"object","data_type":"object","additionalProperties":false},"resourceType":{"type":"object","arrival_format":"object","data_type":"object","additionalProperties":false},"identifier":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"contentType":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"mimeType":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"hashTagId":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"compatibilityLevel":{"type":"object","properties":{"min":{"type":"integer","arrival_format":"number","data_type":"integer"},"max":{"type":"integer","arrival_format":"number","data_type":"integer"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"createdBy":{"type":"string","arrival_format":"text","data_type":"string"},"mediaType":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"origin":{"type":"string","arrival_format":"text","data_type":"string"},"primaryCategory":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"trackable":{"enabled":{"type":"string"}}},"arrival_format":"object","data_type":"object","additionalProperties":false},"sort":{"type":"object","properties":{"lastUpdatedOn":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"topn":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"pageid":{"type":"string","arrival_format":"text","data_type":"string"},"uri":{"type":"string","arrival_format":"text","data_type":"string"},"subtype":{"type":"string","arrival_format":"text","data_type":"string"},"id":{"type":"string","arrival_format":"text","data_type":"string"},"data":{"type":"string","arrival_format":"text","data_type":"string"},"uaspec":{"type":"object","properties":{"agent":{"type":"string","arrival_format":"text","data_type":"string"},"ver":{"type":"string","arrival_format":"text","data_type":"string"},"system":{"type":"string","arrival_format":"text","data_type":"string"},"platform":{"type":"string","arrival_format":"text","data_type":"string"},"raw":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"state":{"type":"string","arrival_format":"text","data_type":"string"},"props":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"prevstate":{"type":"string","arrival_format":"text","data_type":"string"},"dspec":{"type":"object","properties":{"os":{"type":"string","arrival_format":"text","data_type":"string"},"make":{"type":"string","arrival_format":"text","data_type":"string"},"id":{"type":"string","arrival_format":"text","data_type":"string"},"idisk":{"type":"number","arrival_format":"number","data_type":"number"},"edisk":{"type":"number","arrival_format":"number","data_type":"number"},"scrn":{"type":"number","arrival_format":"number","data_type":"number"},"camera":{"type":"string","arrival_format":"text","data_type":"string"},"cpu":{"type":"string","arrival_format":"text","data_type":"string"},"sims":{"type":"integer","arrival_format":"number","data_type":"integer"},"webview":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"extra":{"type":"object","properties":{"pos":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"values":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"query":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"mode":{"type":"string","arrival_format":"text","data_type":"string"}},"duration":{"type":"string"},"arrival_format":"object","data_type":"object","additionalProperties":false},"@timestamp":{"type":"string","suggestions":[{"message":"The Property \'@timestamp\' appears to be \'date-time\' format type.","advice":"The System can index all data on this column","resolutionType":"INDEX","severity":"LOW","path":"properties.@timestamp"}],"arrival_format":"text","data_type":"date-time"},"context":{"type":"object","properties":{"channel":{"type":"string","arrival_format":"text","data_type":"string"},"pdata":{"type":"object","properties":{"id":{"type":"string","arrival_format":"text","data_type":"string"},"ver":{"type":"string","arrival_format":"text","data_type":"string"},"pid":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"env":{"type":"string","arrival_format":"text","data_type":"string"},"sid":{"type":"string","arrival_format":"text","data_type":"string"},"rollup":{"type":"object","properties":{"l1":{"type":"string","arrival_format":"text","data_type":"string"},"l2":{"type":"string","arrival_format":"text","data_type":"string"},"l3":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"cdata":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"},"did":{"type":"string","arrival_format":"text","data_type":"string"},"uid":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"object":{"type":"object","properties":{"id":{"type":"string","arrival_format":"text","data_type":"string"},"type":{"type":"string","arrival_format":"text","data_type":"string"},"ver":{"type":"string","arrival_format":"text","data_type":"string"},"rollup":{"type":"object","properties":{"l1":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"version":{"type":"string","arrival_format":"text","data_type":"string"}},"arrival_format":"object","data_type":"object","additionalProperties":false},"tags":{"type":"array","items":{"type":"object","properties":{"key":{"type":"string"},"value":{"type":"string"}}},"additionalProperties":true,"arrival_format":"array","data_type":"array"}},"additionalProperties":false},"denorm_config":{"redis_db_host":"redis-denorm-headless.redis.svc.cluster.local","redis_db_port":6379,"denorm_fields":[{"denorm_key":"actor.id","dataset_id":"master-dataset","denorm_out_field":"userdata"}]},"router_config":{"topic":"sb-telemetry"},"dataset_config":{"keys_config":{"timestamp_key":"obsrv_meta.syncts","data_key":"","partition_key":""},"indexing_config":{"olap_store_enabled":true,"lakehouse_enabled":false,"cache_enabled":false},"cache_config":{"redis_db_host":"redis-denorm-headless.redis.svc.cluster.local","redis_db_port":6379,"redis_db":0},"file_upload_path":[]},"transformations_config":[],"connectors_config":[],"tags":["tag1"],"status":"ReadyToPublish","version":1,"created_by":"SYSTEM","updated_by":"SYSTEM","version_key":"1724333643940","api_version":"v2","sample_data":{},"entry_topic":"dev.ingest","created_date":"2024-07-23T18:35:15.690Z","updated_date":"2024-08-22T13:34:08.041Z"}}'};
try { const response = await fetch(url, options); const data = await response.json(); console.log(data);} catch (error) { console.error(error);}curl --request POST \ --url 'http://localhost:3000/v2/datasets/import?overwrite=false' \ --header 'Content-Type: application/json' \ --data '{ "id": "api.datasets.import", "ver": "v2", "ts": "2024-05-21T14:30:00Z", "params": { "msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7" }, "request": { "id": "sb-telemetry_draft_data", "dataset_id": "sb-telemetryRPF_draft_data", "name": "sb-telemetry_draft_data", "type": "event", "extraction_config": { "is_batch_event": false, "dedup_config": { "drop_duplicates": false, "dedup_key": "id", "dedup_period": 604800 }, "extraction_key": "events" }, "validation_config": { "validate": true, "mode": "Strict" }, "dedup_config": { "dedup_key": "id", "drop_duplicates": true, "dedup_period": 604800 }, "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "eid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "syncts": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch", "suggestions": [ { "message": "The Property '\''ets'\'' appears to be '\''date-time'\'' format type.", "advice": "The System can index all data on this column", "resolutionType": "INDEX", "severity": "LOW", "path": "properties.ets" } ] }, "flags": { "type": "object", "properties": { "ex_processed": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "pp_validation_processed": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "pp_duplicate_skipped": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "user_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "device_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "loc_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "content_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "coll_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "mid": { "type": "string", "suggestions": [ { "message": "The Property '\''mid'\'' appears to be '\''uuid'\'' format type.", "advice": "Suggest to not to index the high cardinal columns", "resolutionType": "DEDUP", "severity": "LOW", "path": "properties.mid" } ], "arrival_format": "text", "data_type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string", "suggestions": [ { "message": "The Property '\''actor.id'\'' appears to be '\''uuid'\'' format type.", "advice": "Suggest to not to index the high cardinal columns", "resolutionType": "DEDUP", "severity": "LOW", "path": "properties.actor.properties.id" } ], "arrival_format": "text", "data_type": "string" }, "type": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "edata": { "type": "object", "properties": { "visits": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "type": { "type": "string", "arrival_format": "text", "data_type": "string" }, "duration": { "type": "number", "arrival_format": "number", "data_type": "number" }, "size": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "query": { "type": "string", "arrival_format": "text", "data_type": "string" }, "filters": { "type": "object", "properties": { "objectType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "version": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "status": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "id": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "isRootOrg": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "trackable.enabled": { "type": "string", "arrival_format": "text", "data_type": "string" }, "channel": { "type": "object", "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "framework": { "type": "object", "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "resourceType": { "type": "object", "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "identifier": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "contentType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "mimeType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "hashTagId": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "compatibilityLevel": { "type": "object", "properties": { "min": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "max": { "type": "integer", "arrival_format": "number", "data_type": "integer" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "createdBy": { "type": "string", "arrival_format": "text", "data_type": "string" }, "mediaType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "origin": { "type": "string", "arrival_format": "text", "data_type": "string" }, "primaryCategory": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "trackable": { "enabled": { "type": "string" } } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "sort": { "type": "object", "properties": { "lastUpdatedOn": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "topn": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "pageid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "uri": { "type": "string", "arrival_format": "text", "data_type": "string" }, "subtype": { "type": "string", "arrival_format": "text", "data_type": "string" }, "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "data": { "type": "string", "arrival_format": "text", "data_type": "string" }, "uaspec": { "type": "object", "properties": { "agent": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "system": { "type": "string", "arrival_format": "text", "data_type": "string" }, "platform": { "type": "string", "arrival_format": "text", "data_type": "string" }, "raw": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "state": { "type": "string", "arrival_format": "text", "data_type": "string" }, "props": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "prevstate": { "type": "string", "arrival_format": "text", "data_type": "string" }, "dspec": { "type": "object", "properties": { "os": { "type": "string", "arrival_format": "text", "data_type": "string" }, "make": { "type": "string", "arrival_format": "text", "data_type": "string" }, "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "idisk": { "type": "number", "arrival_format": "number", "data_type": "number" }, "edisk": { "type": "number", "arrival_format": "number", "data_type": "number" }, "scrn": { "type": "number", "arrival_format": "number", "data_type": "number" }, "camera": { "type": "string", "arrival_format": "text", "data_type": "string" }, "cpu": { "type": "string", "arrival_format": "text", "data_type": "string" }, "sims": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "webview": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "extra": { "type": "object", "properties": { "pos": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "values": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "query": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "mode": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "duration": { "type": "string" }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "@timestamp": { "type": "string", "suggestions": [ { "message": "The Property '\''@timestamp'\'' appears to be '\''date-time'\'' format type.", "advice": "The System can index all data on this column", "resolutionType": "INDEX", "severity": "LOW", "path": "properties.@timestamp" } ], "arrival_format": "text", "data_type": "date-time" }, "context": { "type": "object", "properties": { "channel": { "type": "string", "arrival_format": "text", "data_type": "string" }, "pdata": { "type": "object", "properties": { "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "pid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "env": { "type": "string", "arrival_format": "text", "data_type": "string" }, "sid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "rollup": { "type": "object", "properties": { "l1": { "type": "string", "arrival_format": "text", "data_type": "string" }, "l2": { "type": "string", "arrival_format": "text", "data_type": "string" }, "l3": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "cdata": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "did": { "type": "string", "arrival_format": "text", "data_type": "string" }, "uid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "object": { "type": "object", "properties": { "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "type": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "rollup": { "type": "object", "properties": { "l1": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "version": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "tags": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" } }, "additionalProperties": false }, "denorm_config": { "redis_db_host": "redis-denorm-headless.redis.svc.cluster.local", "redis_db_port": 6379, "denorm_fields": [ { "denorm_key": "actor.id", "dataset_id": "master-dataset", "denorm_out_field": "userdata" } ] }, "router_config": { "topic": "sb-telemetry" }, "dataset_config": { "keys_config": { "timestamp_key": "obsrv_meta.syncts", "data_key": "", "partition_key": "" }, "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "cache_config": { "redis_db_host": "redis-denorm-headless.redis.svc.cluster.local", "redis_db_port": 6379, "redis_db": 0 }, "file_upload_path": [] }, "transformations_config": [], "connectors_config": [], "tags": [ "tag1" ], "status": "ReadyToPublish", "version": 1, "created_by": "SYSTEM", "updated_by": "SYSTEM", "version_key": "1724333643940", "api_version": "v2", "sample_data": {}, "entry_topic": "dev.ingest", "created_date": "2024-07-23T18:35:15.690Z", "updated_date": "2024-08-22T13:34:08.041Z" } }'Parameters
Section titled “ Parameters ”Query Parameters
Section titled “Query Parameters ” overwrite
boolean
Example
falseRequest Body
Section titled “Request Body ” Media type application/json
object
Example
{ "id": "api.datasets.import", "ver": "v2", "ts": "2024-05-21T14:30:00Z", "params": { "msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7" }, "request": { "id": "sb-telemetry_draft_data", "dataset_id": "sb-telemetryRPF_draft_data", "name": "sb-telemetry_draft_data", "type": "event", "extraction_config": { "is_batch_event": false, "dedup_config": { "drop_duplicates": false, "dedup_key": "id", "dedup_period": 604800 }, "extraction_key": "events" }, "validation_config": { "validate": true, "mode": "Strict" }, "dedup_config": { "dedup_key": "id", "drop_duplicates": true, "dedup_period": 604800 }, "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "eid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "syncts": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch", "suggestions": [ { "message": "The Property 'ets' appears to be 'date-time' format type.", "advice": "The System can index all data on this column", "resolutionType": "INDEX", "severity": "LOW", "path": "properties.ets" } ] }, "flags": { "type": "object", "properties": { "ex_processed": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "pp_validation_processed": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "pp_duplicate_skipped": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "user_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "device_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "loc_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "content_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "coll_denorm": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "mid": { "type": "string", "suggestions": [ { "message": "The Property 'mid' appears to be 'uuid' format type.", "advice": "Suggest to not to index the high cardinal columns", "resolutionType": "DEDUP", "severity": "LOW", "path": "properties.mid" } ], "arrival_format": "text", "data_type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string", "suggestions": [ { "message": "The Property 'actor.id' appears to be 'uuid' format type.", "advice": "Suggest to not to index the high cardinal columns", "resolutionType": "DEDUP", "severity": "LOW", "path": "properties.actor.properties.id" } ], "arrival_format": "text", "data_type": "string" }, "type": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "edata": { "type": "object", "properties": { "visits": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "type": { "type": "string", "arrival_format": "text", "data_type": "string" }, "duration": { "type": "number", "arrival_format": "number", "data_type": "number" }, "size": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "query": { "type": "string", "arrival_format": "text", "data_type": "string" }, "filters": { "type": "object", "properties": { "objectType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "version": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "status": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "id": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "isRootOrg": { "type": "boolean", "arrival_format": "boolean", "data_type": "boolean" }, "trackable.enabled": { "type": "string", "arrival_format": "text", "data_type": "string" }, "channel": { "type": "object", "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "framework": { "type": "object", "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "resourceType": { "type": "object", "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "identifier": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "contentType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "mimeType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "hashTagId": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "compatibilityLevel": { "type": "object", "properties": { "min": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "max": { "type": "integer", "arrival_format": "number", "data_type": "integer" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "createdBy": { "type": "string", "arrival_format": "text", "data_type": "string" }, "mediaType": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "origin": { "type": "string", "arrival_format": "text", "data_type": "string" }, "primaryCategory": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "trackable": { "enabled": { "type": "string" } } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "sort": { "type": "object", "properties": { "lastUpdatedOn": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "topn": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "pageid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "uri": { "type": "string", "arrival_format": "text", "data_type": "string" }, "subtype": { "type": "string", "arrival_format": "text", "data_type": "string" }, "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "data": { "type": "string", "arrival_format": "text", "data_type": "string" }, "uaspec": { "type": "object", "properties": { "agent": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "system": { "type": "string", "arrival_format": "text", "data_type": "string" }, "platform": { "type": "string", "arrival_format": "text", "data_type": "string" }, "raw": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "state": { "type": "string", "arrival_format": "text", "data_type": "string" }, "props": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "prevstate": { "type": "string", "arrival_format": "text", "data_type": "string" }, "dspec": { "type": "object", "properties": { "os": { "type": "string", "arrival_format": "text", "data_type": "string" }, "make": { "type": "string", "arrival_format": "text", "data_type": "string" }, "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "idisk": { "type": "number", "arrival_format": "number", "data_type": "number" }, "edisk": { "type": "number", "arrival_format": "number", "data_type": "number" }, "scrn": { "type": "number", "arrival_format": "number", "data_type": "number" }, "camera": { "type": "string", "arrival_format": "text", "data_type": "string" }, "cpu": { "type": "string", "arrival_format": "text", "data_type": "string" }, "sims": { "type": "integer", "arrival_format": "number", "data_type": "integer" }, "webview": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "extra": { "type": "object", "properties": { "pos": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "values": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "query": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "mode": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "duration": { "type": "string" }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "@timestamp": { "type": "string", "suggestions": [ { "message": "The Property '@timestamp' appears to be 'date-time' format type.", "advice": "The System can index all data on this column", "resolutionType": "INDEX", "severity": "LOW", "path": "properties.@timestamp" } ], "arrival_format": "text", "data_type": "date-time" }, "context": { "type": "object", "properties": { "channel": { "type": "string", "arrival_format": "text", "data_type": "string" }, "pdata": { "type": "object", "properties": { "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "pid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "env": { "type": "string", "arrival_format": "text", "data_type": "string" }, "sid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "rollup": { "type": "object", "properties": { "l1": { "type": "string", "arrival_format": "text", "data_type": "string" }, "l2": { "type": "string", "arrival_format": "text", "data_type": "string" }, "l3": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "cdata": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" }, "did": { "type": "string", "arrival_format": "text", "data_type": "string" }, "uid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "object": { "type": "object", "properties": { "id": { "type": "string", "arrival_format": "text", "data_type": "string" }, "type": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ver": { "type": "string", "arrival_format": "text", "data_type": "string" }, "rollup": { "type": "object", "properties": { "l1": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "version": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "arrival_format": "object", "data_type": "object", "additionalProperties": false }, "tags": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string" }, "value": { "type": "string" } } }, "additionalProperties": true, "arrival_format": "array", "data_type": "array" } }, "additionalProperties": false }, "denorm_config": { "redis_db_host": "redis-denorm-headless.redis.svc.cluster.local", "redis_db_port": 6379, "denorm_fields": [ { "denorm_key": "actor.id", "dataset_id": "master-dataset", "denorm_out_field": "userdata" } ] }, "router_config": { "topic": "sb-telemetry" }, "dataset_config": { "keys_config": { "timestamp_key": "obsrv_meta.syncts", "data_key": "", "partition_key": "" }, "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "cache_config": { "redis_db_host": "redis-denorm-headless.redis.svc.cluster.local", "redis_db_port": 6379, "redis_db": 0 }, "file_upload_path": [] }, "transformations_config": [], "connectors_config": [], "tags": [ "tag1" ], "status": "ReadyToPublish", "version": 1, "created_by": "SYSTEM", "updated_by": "SYSTEM", "version_key": "1724333643940", "api_version": "v2", "sample_data": {}, "entry_topic": "dev.ingest", "created_date": "2024-07-23T18:35:15.690Z", "updated_date": "2024-08-22T13:34:08.041Z" }}Responses
Section titled “ Responses ”OK
Media type application/json
object
Examples
Success: Fully imported dataset
{ "id": "api.datasets.import", "ver": "v2", "ts": "2024-11-19T13:04:26+05:30", "params": { "status": "SUCCESS", "msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7", "resmsgid": "fd87393e-4b1b-4cf0-8e9a-a0529d691cc3" }, "responseCode": "OK", "result": { "message": "Dataset is imported successfully", "data": { "id": "sb-telemetry_draft_data", "version_key": "1732001666671" } }}Success: Partially imported dataset
{ "id": "api.datasets.import", "ver": "v2", "ts": "2024-11-19T12:59:41+05:30", "params": { "status": "SUCCESS", "msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7", "resmsgid": "379ced0b-eb06-445f-a7e3-8737cb16c351" }, "responseCode": "OK", "result": { "message": "Dataset is partially imported", "data": { "id": "sb-telemetry_draft_data", "version_key": "1732001381825" }, "ignoredFields": { "denorm_fields": [ { "config": { "denorm_key": "actor.id", "denorm_out_field": "userdata", "dataset_id": "master-dataset" }, "details": "Master dataset does not exist" } ] } }}Success: Imported new dataset [overwrite=false]
{ "id": "api.datasets.import", "ver": "v2", "ts": "2024-11-19T13:05:32+05:30", "params": { "status": "SUCCESS", "msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7", "resmsgid": "1a50e1b1-14b0-4178-aba4-9e09a63d8d1d" }, "responseCode": "OK", "result": { "message": "Dataset is imported successfully", "data": { "id": "sb-telemetry_draft", "version_key": "1732001732045" } }}Headers
Section titled “Headers ” X-Powered-By
string
Example
Express Content-Length
integer
Example
346 ETag
string
Example
W/"15a-SX5o6plRdfy+akEYhbMwq9zl+oU" Date
string
Example
Tue, 19 Nov 2024 07:34:26 GMT Connection
string
Example
keep-alive Keep-Alive
string
Example
timeout=5Bad Request
Media type application/json
object
Example
{ "id": "api.datasets.import", "ver": "v2", "ts": "2024-11-19T13:08:05+05:30", "params": { "status": "FAILED", "msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7", "resmsgid": "244318a5-d933-41a5-add6-14f299281069" }, "responseCode": "BAD_REQUEST", "result": {}, "error": { "code": "DATASET_IMPORT_INVALID_CONFIGS", "message": "#properties/request/required must have required property 'id'" }}Headers
Section titled “Headers ” X-Powered-By
string
Example
Express Content-Length
integer
Example
364 ETag
string
Example
W/"16c-Dxem7prJM89DOqth+bdZbh0UI1g" Date
string
Example
Tue, 19 Nov 2024 07:38:05 GMT Connection
string
Example
keep-alive Keep-Alive
string
Example
timeout=5Conflict
Media type application/json
object
Example
{ "id": "api.datasets.import", "ver": "v2", "ts": "2024-11-19T13:01:11+05:30", "params": { "status": "FAILED", "msgid": "127384e4a-a051-4a9f-9b3f-a64a8034fad7", "resmsgid": "1e5b34cd-4473-4439-8cc5-5ff917f47e48" }, "responseCode": "CONFLICT", "result": {}, "error": { "code": "DATASET_EXISTS", "message": "Dataset with dataset_id: sb-telemetryRPF_draft_data already exists." }}Headers
Section titled “Headers ” X-Powered-By
string
Example
Express Content-Length
integer
Example
351 ETag
string
Example
W/"15f-nj5WJ+wiNbI5ViTPWUzGutdBSX4" Date
string
Example
Tue, 19 Nov 2024 07:31:11 GMT Connection
string
Example
keep-alive Keep-Alive
string
Example
timeout=5