Skip to content

Create Dataset

POST
/v2/datasets/create
 
curl --request POST \
  --url http://localhost:3000/v2/datasets/create \
  --header 'Content-Type: application/json' \
  --data '{ "id": "api.datasets.create", "ver": "v2", "ts": "2024-04-10T16:10:50+05:30", "params": { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { "dataset_id": "telemetry_record-t4", "type": "event", "name": "sb-telemetry", "validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id" } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "mid" }, "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, "denorm_config": { "denorm_fields": [ { "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-telemetry" } ] }, "transformations_config": [ { "field_key": "email", "transformation_function": { "type": "mask", "expr": "mid", "datatype": "string", "category": "pii" }, "mode": "Strict" } ], "tags": [ "tag1" ] } }'

This API allows you to create new datasets used by the analytical data source.

Request Body

Section titled “Request Body ”
Media type application/json
object
Example 
{
  "id": "api.datasets.create",
  "ver": "v2",
  "ts": "2024-04-10T16:10:50+05:30",
  "params": {
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
  },
  "request": {
    "dataset_id": "telemetry_record-t4",
    "type": "event",
    "name": "sb-telemetry",
    "validation_config": {
      "validate": true,
      "mode": "Strict"
    },
    "extraction_config": {
      "is_batch_event": true,
      "extraction_key": "events",
      "dedup_config": {
        "drop_duplicates": true,
        "dedup_key": "id"
      }
    },
    "dedup_config": {
      "drop_duplicates": true,
      "dedup_key": "mid"
    },
    "data_schema": {
      "$schema": "https://json-schema.org/draft/2020-12/schema",
      "type": "object",
      "properties": {
        "mid": {
          "type": "string",
          "arrival_format": "text",
          "data_type": "string"
        },
        "ets": {
          "type": "integer",
          "arrival_format": "number",
          "data_type": "epoch"
        },
        "eid": {
          "type": "string",
          "arrival_format": "text",
          "data_type": "string"
        }
      },
      "additionalProperties": true
    },
    "denorm_config": {
      "denorm_fields": [
        {
          "denorm_key": "eid",
          "denorm_out_field": "userdata",
          "dataset_id": "master-telemetry"
        }
      ]
    },
    "transformations_config": [
      {
        "field_key": "email",
        "transformation_function": {
          "type": "mask",
          "expr": "mid",
          "datatype": "string",
          "category": "pii"
        },
        "mode": "Strict"
      }
    ],
    "tags": [
      "tag1"
    ]
  }
}

Responses

Section titled “ Responses ”

200

Section titled “200 ”

OK

Media type application/json
object
Examples

Success: Dataset created successfullly

{
  "id": "api.datasets.create",
  "ver": "v2",
  "ts": "2024-07-15T18:44:08+05:30",
  "params": {
    "status": "SUCCESS",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "276c042c-0f23-4b26-9b10-6fe48bbc2d3a"
  },
  "responseCode": "OK",
  "result": {
    "id": "telemetry_record-t4",
    "version_key": "1721049248930"
  }
}

Headers

Section titled “Headers ”
X-Powered-By
string
Example 
Express
Content-Length
integer
Example 
287
ETag
string
Example 
W/"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ"
Date
string
Example 
Mon, 15 Jul 2024 13:14:09 GMT
Connection
string
Example 
keep-alive
Keep-Alive
string
Example 
timeout=5

400

Section titled “400 ”

Bad Request

Media type application/json
object
Example 
{
  "id": "api.datasets.create",
  "ver": "v2",
  "ts": "2024-07-16T08:39:00+05:30",
  "params": {
    "status": "FAILED",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "a07de860-dcbc-4ff6-822e-34b47635c8a3"
  },
  "responseCode": "BAD_REQUEST",
  "result": {},
  "error": {
    "code": "DATASET_INVALID_INPUT",
    "message": "#properties/request/required must have required property 'dataset_id'"
  }
}

Headers

Section titled “Headers ”
X-Powered-By
string
Example 
Express
Content-Length
integer
Example 
362
ETag
string
Example 
W/"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI"
Date
string
Example 
Tue, 16 Jul 2024 03:09:00 GMT
Connection
string
Example 
keep-alive
Keep-Alive
string
Example 
timeout=5

409

Section titled “409 ”

Conflict

Media type application/json
object
Examples

Failure: Master dataset already exists

{
  "id": "api.files.generate-url",
  "ver": "v2",
  "ts": "2024-07-16T08:37:28+05:30",
  "params": {
    "status": "FAILED",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "138b796b-1b68-481a-a59d-1cb695c1adc9"
  },
  "responseCode": "CONFLICT",
  "result": {},
  "error": {
    "code": "DATASET_EXISTS",
    "message": "Dataset Already exists with id:telemetry_record-master"
  }
}

Headers

Section titled “Headers ”
X-Powered-By
string
Example 
Express
Content-Length
integer
Example 
337
ETag
string
Example 
W/"151-a7dJ9XBUyT3AXNxl1TPcraxMX08"
Date
string
Example 
Tue, 16 Jul 2024 03:07:28 GMT
Connection
string
Example 
keep-alive
Keep-Alive
string
Example 
timeout=5