Update Dataset

PATCH

/v2/datasets/update

const url = 'http://localhost:3000/v2/datasets/update';
const options = {
  method: 'PATCH',
  headers: {'Content-Type': 'application/json'},
  body: '{"id":"api.datasets.update","ver":"v2","ts":"2024-04-10T16:10:50+05:30","params":{"msgid":"4a7f14c3-d61e-4d4f-be78-181834eeff6d"},"request":{"dataset_id":"telemetry_record-t4","version_key":"1721135455988","name":"sb-telemetry","validation_config":{"validate":true,"mode":"Strict"},"extraction_config":{"is_batch_event":true,"extraction_key":"events","dedup_config":{"drop_duplicates":true,"dedup_key":"ipid"}},"dedup_config":{"drop_duplicates":true,"dedup_key":"mid"},"data_schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"midpid":{"type":"string","arrival_format":"text","data_type":"string"},"miduwi":{"type":"integer","arrival_format":"number","data_type":"epoch"},"mid":{"type":"string","arrival_format":"text","data_type":"string"},"sid":{"type":"string","arrival_format":"text","data_type":"string"}},"additionalProperties":true},"denorm_config":{"denorm_fields":[{"value":{"denorm_key":"eid","denorm_out_field":"userdata"},"action":"remove"},{"value":{"denorm_key":"eid","denorm_out_field":"edata","dataset_id":"trip-details"},"action":"upsert"}]},"transformations_config":[{"value":{"field_key":"email","transformation_function":{"type":"mask","expr":"mid","datatype":"string","category":"pii"},"mode":"Strict"},"action":"upsert"},{"value":{"field_key":"email_id","transformation_function":{"type":"mask","expr":"mid","datatype":"string","category":"pii"},"mode":"Strict"},"action":"remove"}],"tags":[],"connectors_config":[{"value":{"connector_id":"jdbc","connector_config":{"source_database_type":"postgresql","source_database_host":"postgresql-hl.postgresql.svc.cluster.local.master","source_database_port":5432,"source_database_name":"obsrv_sample_datasets_1","source_database_username":"postgres","source_database_pwd":"postgres","table":"new_york_taxi_data","timestamp-column":"tpep_pickup_datetime","batch-size":100,"max-batches":2},"operations_config":{"polling_interval":"periodic","schedule":"twice"}},"action":"upsert"}]}}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}

curl --request PATCH \
  --url http://localhost:3000/v2/datasets/update \
  --header 'Content-Type: application/json' \
  --data '{ "id": "api.datasets.update", "ver": "v2", "ts": "2024-04-10T16:10:50+05:30", "params": { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { "dataset_id": "telemetry_record-t4", "version_key": "1721135455988", "name": "sb-telemetry", "validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "ipid" } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "mid" }, "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "midpid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "miduwi": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "sid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, "denorm_config": { "denorm_fields": [ { "value": { "denorm_key": "eid", "denorm_out_field": "userdata" }, "action": "remove" }, { "value": { "denorm_key": "eid", "denorm_out_field": "edata", "dataset_id": "trip-details" }, "action": "upsert" } ] }, "transformations_config": [ { "value": { "field_key": "email", "transformation_function": { "type": "mask", "expr": "mid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }, { "value": { "field_key": "email_id", "transformation_function": { "type": "mask", "expr": "mid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "remove" } ], "tags": [], "connectors_config": [ { "value": { "connector_id": "jdbc", "connector_config": { "source_database_type": "postgresql", "source_database_host": "postgresql-hl.postgresql.svc.cluster.local.master", "source_database_port": 5432, "source_database_name": "obsrv_sample_datasets_1", "source_database_username": "postgres", "source_database_pwd": "postgres", "table": "new_york_taxi_data", "timestamp-column": "tpep_pickup_datetime", "batch-size": 100, "max-batches": 2 }, "operations_config": { "polling_interval": "periodic", "schedule": "twice" } }, "action": "upsert" } ] } }'

This API allows you to update existing datasets, add or remove denorm fields used by the analytical data source. User can even add, remove or update transformations and connectors

Request Body

application/json

object

Example

{
  "id": "api.datasets.update",
  "ver": "v2",
  "ts": "2024-04-10T16:10:50+05:30",
  "params": {
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d"
  },
  "request": {
    "dataset_id": "telemetry_record-t4",
    "version_key": "1721135455988",
    "name": "sb-telemetry",
    "validation_config": {
      "validate": true,
      "mode": "Strict"
    },
    "extraction_config": {
      "is_batch_event": true,
      "extraction_key": "events",
      "dedup_config": {
        "drop_duplicates": true,
        "dedup_key": "ipid"
      }
    },
    "dedup_config": {
      "drop_duplicates": true,
      "dedup_key": "mid"
    },
    "data_schema": {
      "$schema": "https://json-schema.org/draft/2020-12/schema",
      "type": "object",
      "properties": {
        "midpid": {
          "type": "string",
          "arrival_format": "text",
          "data_type": "string"
        },
        "miduwi": {
          "type": "integer",
          "arrival_format": "number",
          "data_type": "epoch"
        },
        "mid": {
          "type": "string",
          "arrival_format": "text",
          "data_type": "string"
        },
        "sid": {
          "type": "string",
          "arrival_format": "text",
          "data_type": "string"
        }
      },
      "additionalProperties": true
    },
    "denorm_config": {
      "denorm_fields": [
        {
          "value": {
            "denorm_key": "eid",
            "denorm_out_field": "userdata"
          },
          "action": "remove"
        },
        {
          "value": {
            "denorm_key": "eid",
            "denorm_out_field": "edata",
            "dataset_id": "trip-details"
          },
          "action": "upsert"
        }
      ]
    },
    "transformations_config": [
      {
        "value": {
          "field_key": "email",
          "transformation_function": {
            "type": "mask",
            "expr": "mid",
            "datatype": "string",
            "category": "pii"
          },
          "mode": "Strict"
        },
        "action": "upsert"
      },
      {
        "value": {
          "field_key": "email_id",
          "transformation_function": {
            "type": "mask",
            "expr": "mid",
            "datatype": "string",
            "category": "pii"
          },
          "mode": "Strict"
        },
        "action": "remove"
      }
    ],
    "tags": [],
    "connectors_config": [
      {
        "value": {
          "connector_id": "jdbc",
          "connector_config": {
            "source_database_type": "postgresql",
            "source_database_host": "postgresql-hl.postgresql.svc.cluster.local.master",
            "source_database_port": 5432,
            "source_database_name": "obsrv_sample_datasets_1",
            "source_database_username": "postgres",
            "source_database_pwd": "postgres",
            "table": "new_york_taxi_data",
            "timestamp-column": "tpep_pickup_datetime",
            "batch-size": 100,
            "max-batches": 2
          },
          "operations_config": {
            "polling_interval": "periodic",
            "schedule": "twice"
          }
        },
        "action": "upsert"
      }
    ]
  }
}

Responses

200

application/json

object

Examples

Success: Minimal dataset update

{
  "id": "api.datasets.update",
  "ver": "v2",
  "ts": "2024-07-16T18:30:45+05:30",
  "params": {
    "status": "SUCCESS",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "354f1fec-0c39-42ee-a52a-49552f847c11"
  },
  "responseCode": "OK",
  "result": {
    "message": "Dataset is updated successfully",
    "id": "telemetry_record-t4",
    "version_key": "1721134845559"
  }
}

Success: Updated successfully

{
  "id": "api.datasets.update",
  "ver": "v2",
  "ts": "2024-07-16T18:27:55+05:30",
  "params": {
    "status": "SUCCESS",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "6d835f07-aed5-4e8b-81c2-2142cfb55c52"
  },
  "responseCode": "OK",
  "result": {
    "message": "Dataset is updated successfully",
    "id": "telemetry_record-t4",
    "version_key": "1721134675878"
  }
}

Headers

X-Powered-By

string

Example

Express

Content-Length

integer

Example

ETag

string

Example

W/"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo"

Date

string

Example

Tue, 16 Jul 2024 13:00:45 GMT

Connection

string

Example

keep-alive

Keep-Alive

string

Example

timeout=5

400

Bad Request

application/json

object

Examples

Failure: Invalid request

{
  "id": "api.datasets.update",
  "ver": "v2",
  "ts": "2024-07-16T18:29:21+05:30",
  "params": {
    "status": "FAILED",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "7d31672b-e5c3-4a6d-afac-d9d78011bcde"
  },
  "responseCode": "BAD_REQUEST",
  "result": {},
  "error": {
    "code": "DATASET_UPDATE_INPUT_INVALID",
    "message": "#properties/request/required must have required property 'dataset_id'"
  }
}

Failure: No fields are provided to update

{
  "id": "api.datasets.update",
  "ver": "v2",
  "ts": "2024-07-16T18:32:44+05:30",
  "params": {
    "status": "FAILED",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "bf99b1e1-7694-4be0-ba5d-e347764736de"
  },
  "responseCode": "BAD_REQUEST",
  "result": {},
  "error": {
    "code": "DATASET_UPDATE_NO_FIELDS",
    "message": "Provide atleast one field in addition to the dataset_id to update the dataset"
  }
}

Headers

X-Powered-By

string

Example

Express

Content-Length

integer

Example

ETag

string

Example

W/"171-iNJoyWUecOEsXbHZwx6rld3Sr1I"

Date

string

Example

Tue, 16 Jul 2024 12:59:21 GMT

Connection

string

Example

keep-alive

Keep-Alive

string

Example

timeout=5

404

Not Found

application/json

object

Example

{
  "id": "api.datasets.update",
  "ver": "v2",
  "ts": "2024-07-16T18:28:30+05:30",
  "params": {
    "status": "FAILED",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "bf64703c-bb6b-41bf-bc1a-c85373efd925"
  },
  "responseCode": "NOT_FOUND",
  "result": {},
  "error": {
    "code": "DATASET_NOT_EXISTS",
    "message": "Dataset does not exists with id:telemetry_record-t41"
  }
}

Headers

X-Powered-By

string

Example

Express

Content-Length

integer

Example

ETag

string

Example

W/"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY"

Date

string

Example

Tue, 16 Jul 2024 12:58:30 GMT

Connection

string

Example

keep-alive

Keep-Alive

string

Example

timeout=5

409

Conflict

application/json

object

Example

{
  "id": "api.datasets.update",
  "ver": "v2",
  "ts": "2024-07-16T18:23:16+05:30",
  "params": {
    "status": "FAILED",
    "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d",
    "resmsgid": "02fe03f6-c4c4-48f6-9d84-a32cd52f4c13"
  },
  "responseCode": "CONFLICT",
  "result": {},
  "error": {
    "code": "DATASET_OUTDATED",
    "message": "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates"
  }
}

Headers

X-Powered-By

string

Example

Express

Content-Length

integer

Example

ETag

string

Example

W/"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw"

Date

string

Example

Tue, 16 Jul 2024 12:53:16 GMT

Connection

string

Example

keep-alive

Keep-Alive

string

Example

timeout=5