LogoLogo
Release notesAPI docsDocs homeStructural CloudTonic.ai
  • Tonic Structural User Guide
  • About Tonic Structural
    • Structural data generation workflow
    • Structural deployment types
    • Structural implementation roles
    • Structural license plans
  • Logging into Structural for the first time
  • Getting started with the Structural free trial
  • Managing your user account
  • Frequently Asked Questions
  • Tutorial videos
  • Creating and managing workspaces
    • Managing workspaces
      • Viewing your list of workspaces
      • Creating, editing, or deleting a workspace
      • Workspace configuration settings
        • Workspace identification and connection type
        • Data connection settings
        • Configuring secrets managers for database connections
        • Data generation settings
        • Enabling and configuring upsert
        • Writing output to Tonic Ephemeral
        • Writing output to a container repository
        • Advanced workspace overrides
      • About the workspace management view
      • About workspace inheritance
      • Assigning tags to a workspace
      • Exporting and importing the workspace configuration
    • Managing access to workspaces
      • Sharing workspace access
      • Transferring ownership of a workspace
    • Viewing workspace jobs and job details
  • Configuring data generation
    • Privacy Hub
    • Database View
      • Viewing and configuring tables
      • Viewing the column list
      • Displaying sample data for a column
      • Configuring an individual column
      • Configuring multiple columns
      • Identifying similar columns
      • Commenting on columns
    • Table View
    • Working with document-based data
      • Performing scans on collections
      • Using Collection View
    • Identifying sensitive data
      • Running the Structural sensitivity scan
      • Manually indicating whether a column is sensitive
      • Built-in sensitivity types that Structural detects
      • Creating and managing custom sensitivity rules
    • Table modes
    • Generator information
      • Generator summary
      • Generator reference
        • Address
        • Algebraic
        • Alphanumeric String Key
        • Array Character Scramble
        • Array JSON Mask
        • Array Regex Mask
        • ASCII Key
        • Business Name
        • Categorical
        • Character Scramble
        • Character Substitution
        • Company Name
        • Conditional
        • Constant
        • Continuous
        • Cross Table Sum
        • CSV Mask
        • Custom Categorical
        • Date Truncation
        • Email
        • Event Timestamps
        • File Name
        • Find and Replace
        • FNR
        • Geo
        • HIPAA Address
        • Hostname
        • HStore Mask
        • HTML Mask
        • Integer Key
        • International Address
        • IP Address
        • JSON Mask
        • MAC Address
        • Mongo ObjectId Key
        • Name
        • Noise Generator
        • Null
        • Numeric String Key
        • Passthrough
        • Phone
        • Random Boolean
        • Random Double
        • Random Hash
        • Random Integer
        • Random Timestamp
        • Random UUID
        • Regex Mask
        • Sequential Integer
        • Shipping Container
        • SIN
        • SSN
        • Struct Mask
        • Timestamp Shift Generator
        • Unique Email
        • URL
        • UUID Key
        • XML Mask
      • Generator characteristics
        • Enabling consistency
        • Linking generators
        • Differential privacy
        • Partitioning a column
        • Data-free generators
        • Supporting uniqueness constraints
        • Format-preserving encryption (FPE)
      • Generator types
        • Composite generators
        • Primary key generators
    • Generator assignment and configuration
      • Reviewing and applying recommended generators
      • Assigning and configuring generators
      • Document View for file connector JSON columns
      • Generator hints and tips
      • Managing generator presets
      • Configuring and using Structural data encryption
      • Custom value processors
    • Subsetting data
      • About subsetting
      • Using table filtering for data warehouses and Spark-based data connectors
      • Viewing the current subsetting configuration
      • Subsetting and foreign keys
      • Configuring subsetting
      • Viewing and managing configuration inheritance
      • Viewing the subset creation steps
      • Viewing previous subsetting data generation runs
      • Generating cohesive subset data from related databases
      • Other subsetting hints and tips
    • Viewing and adding foreign keys
    • Viewing and resolving schema changes
    • Tracking changes to workspaces, generator presets, and sensitivity rules
    • Using the Privacy Report to verify data protection
  • Running data generation
    • Running data generation jobs
      • Types of data generation
      • Data generation process
      • Running data generation manually
      • Scheduling data generation
      • Issues that prevent data generation
    • Managing data generation performance
    • Viewing and downloading container artifacts
    • Post-job scripts
    • Webhooks
  • Installing and Administering Structural
    • Structural architecture
    • Using Structural securely
    • Deploying a self-hosted Structural instance
      • Deployment checklist
      • System requirements
      • Deploying with Docker Compose
      • Deploying on Kubernetes with Helm
      • Enabling the option to write output data to a container repository
        • Setting up a Kubernetes cluster to use to write output data to a container repository
        • Required access to write destination data to a container repository
      • Entering and updating your license key
      • Setting up host integration
      • Working with the application database
      • Setting up a secret
      • Setting a custom certificate
    • Using Structural Cloud
      • Structural Cloud notes
      • Setting up and managing a Structural Cloud pay-as-you-go subscription
      • Structural Cloud onboarding
    • Managing user access to Structural
      • Structural organizations
      • Determining whether users can create accounts
      • Creating a new account in an existing organization
      • Single sign-on (SSO)
        • Structural user authentication with SSO
        • Enabling and configuring SSO on Structural Cloud
        • Synchronizing SSO groups with Structural
        • Viewing the list of SSO groups in Tonic Structural
        • AWS IAM Identity Center
        • Duo
        • GitHub
        • Google
        • Keycloak
        • Microsoft Entra ID (previously Azure Active Directory)
        • Okta
        • OpenID Connect (OIDC)
        • SAML
      • Managing Structural users
      • Managing permissions
        • About permission sets
        • Built-in permission sets
        • Available permissions
        • Viewing the lists of global and workspace permission sets
        • Configuring custom permission sets
        • Selecting default permission sets
        • Configuring access to global permission sets
        • Setting initial access to all global permissions
        • Granting Account Admin access for a Structural Cloud organization
    • Structural monitoring and logging
      • Monitoring Structural services
      • Performing health checks
      • Downloading the usage report
      • Tracking user access and permissions
      • Redacted and diagnostic (unredacted) logs
      • Data that Tonic.ai collects
      • Verifying and enabling telemetry sharing
    • Configuring environment settings
    • Updating Structural
  • Connecting to your data
    • About data connectors
    • Overview for database administrators
    • Data connector summary
    • Amazon DynamoDB
      • System requirements and limitations for DynamoDB
      • Structural differences and limitations with DynamoDB
      • Before you create a DynamoDB workspace
      • Configuring DynamoDB workspace data connections
    • Amazon EMR
      • Structural process overview for Amazon EMR
      • System requirements for Amazon EMR
      • Structural differences and limitations with Amazon EMR
      • Before you create an Amazon EMR workspace
        • Creating IAM roles for Structural and Amazon EMR
        • Creating Athena workgroups
        • Configuration for cross-account setups
      • Configuring Amazon EMR workspace data connections
    • Amazon Redshift
      • Structural process overview for Amazon Redshift
      • Structural differences and limitations with Amazon Redshift
      • Before you create an Amazon Redshift workspace
        • Required AWS instance profile permissions for Amazon Redshift
        • Setting up the AWS Lambda role for Amazon Redshift
        • AWS KMS permissions for Amazon SQS message encryption
        • Amazon Redshift-specific Structural environment settings
        • Source and destination database permissions for Amazon Redshift
      • Configuring Amazon Redshift workspace data connections
    • Databricks
      • Structural process overview for Databricks
      • System requirements for Databricks
      • Structural differences and limitations with Databricks
      • Before you create a Databricks workspace
        • Granting access to storage
        • Setting up your Databricks cluster
        • Configuring the destination database schema creation
      • Configuring Databricks workspace data connections
    • Db2 for LUW
      • System requirements for Db2 for LUW
      • Structural differences and limitations with Db2 for LUW
      • Before you create a Db2 for LUW workspace
      • Configuring Db2 for LUW workspace data connections
    • File connector
      • Overview of the file connector process
      • Supported file and content types
      • Structural differences and limitations with the file connector
      • Before you create a file connector workspace
      • Configuring the file connector storage type and output options
      • Managing file groups in a file connector workspace
      • Downloading generated file connector files
    • Google BigQuery
      • Structural differences and limitations with Google BigQuery
      • Before you create a Google BigQuery workspace
      • Configuring Google BigQuery workspace data connections
      • Resolving schema changes for de-identified views
    • MongoDB
      • System requirements for MongoDB
      • Structural differences and limitations with MongoDB
      • Configuring MongoDB workspace data connections
      • Other MongoDB hints and tips
    • MySQL
      • System requirements for MySQL
      • Before you create a MySQL workspace
      • Configuring MySQL workspace data connections
    • Oracle
      • Known limitations for Oracle schema objects
      • System requirements for Oracle
      • Structural differences and limitations with Oracle
      • Before you create an Oracle workspace
      • Configuring Oracle workspace data connections
    • PostgreSQL
      • System requirements for PostgreSQL
      • Before you create a PostgreSQL workspace
      • Configuring PostgreSQL workspace data connections
    • Salesforce
      • System requirements for Salesforce
      • Structural differences and limitations with Salesforce
      • Before you create a Salesforce workspace
      • Configuring Salesforce workspace data connections
    • Snowflake on AWS
      • Structural process overviews for Snowflake on AWS
      • Structural differences and limitations with Snowflake on AWS
      • Before you create a Snowflake on AWS workspace
        • Required AWS instance profile permissions for Snowflake on AWS
        • Other configuration for Lambda processing
        • Source and destination database permissions for Snowflake on AWS
        • Configuring whether Structural creates the Snowflake on AWS destination database schema
      • Configuring Snowflake on AWS workspace data connections
    • Snowflake on Azure
      • Structural process overview for Snowflake on Azure
      • Structural differences and limitations with Snowflake on Azure
      • Before you create a Snowflake on Azure workspace
      • Configuring Snowflake on Azure workspace data connections
    • Spark SDK
      • Structural process overview for the Spark SDK
      • Structural differences and limitations with the Spark SDK
      • Configuring Spark SDK workspace data connections
      • Using Spark to run de-identification of the data
    • SQL Server
      • System requirements for SQL Server
      • Before you create a SQL Server workspace
      • Configuring SQL Server workspace data connections
    • Yugabyte
      • System requirements for Yugabyte
      • Structural differences and limitations with Yugabyte
      • Before you create a Yugabyte workspace
      • Configuring Yugabyte workspace data connections
      • Troubleshooting Yugabyte data generation issues
  • Using the Structural API
    • About the Structural API
    • Getting an API token
    • Getting the workspace ID
    • Using the Structural API to perform tasks
      • Configure environment settings
      • Manage generator presets
        • Retrieving the list of generator presets
        • Structure of a generator preset
        • Creating a custom generator preset
        • Updating an existing generator preset
        • Deleting a generator preset
      • Manage custom sensitivity rules
      • Create a workspace
      • Connect to source and destination data
      • Manage file groups in a file connector workspace
      • Assign table modes and filters to source database tables
      • Set column sensitivity
      • Assign generators to columns
        • Getting the generator IDs and available metadata
        • Updating generator configurations
        • Structure of a generator assignment
        • Generator API reference
          • Address (AddressGenerator)
          • Algebraic (AlgebraicGenerator)
          • Alphanumeric String Key (AlphaNumericPkGenerator)
          • Array Character Scramble (ArrayTextMaskGenerator)
          • Array JSON Mask (ArrayJsonMaskGenerator)
          • Array Regex Mask (ArrayRegexMaskGenerator)
          • ASCII Key (AsciiPkGenerator)
          • Business Name (BusinessNameGenerator)
          • Categorical (CategoricalGenerator)
          • Character Scramble (TextMaskGenerator)
          • Character Substitution (StringMaskGenerator)
          • Company Name (CompanyNameGenerator)
          • Conditional (ConditionalGenerator)
          • Constant (ConstantGenerator)
          • Continuous (GaussianGenerator)
          • Cross Table Sum (CrossTableAggregateGenerator)
          • CSV Mask (CsvMaskGenerator)
          • Custom Categorical (CustomCategoricalGenerator)
          • Date Truncation (DateTruncationGenerator)
          • Email (EmailGenerator)
          • Event Timestamps (EventGenerator)
          • File Name (FileNameGenerator)
          • Find and Replace (FindAndReplaceGenerator)
          • FNR (FnrGenerator)
          • Geo (GeoGenerator)
          • HIPAA Address (HipaaAddressGenerator)
          • Hostname (HostnameGenerator)
          • HStore Mask (HStoreMaskGenerator)
          • HTML Mask (HtmlMaskGenerator)
          • Integer Key (IntegerPkGenerator)
          • International Address (InternationalAddressGenerator)
          • IP Address (IPAddressGenerator)
          • JSON Mask (JsonMaskGenerator)
          • MAC Address (MACAddressGenerator)
          • Mongo ObjectId Key (ObjectIdPkGenerator)
          • Name (NameGenerator)
          • Noise Generator (NoiseGenerator)
          • Null (NullGenerator)
          • Numeric String Key (NumericStringPkGenerator)
          • Passthrough (PassthroughGenerator)
          • Phone (USPhoneNumberGenerator)
          • Random Boolean (RandomBooleanGenerator)
          • Random Double (RandomDoubleGenerator)
          • Random Hash (RandomStringGenerator)
          • Random Integer (RandomIntegerGenerator)
          • Random Timestamp (RandomTimestampGenerator)
          • Random UUID (UUIDGenerator)
          • Regex Mask (RegexMaskGenerator)
          • Sequential Integer (UniqueIntegerGenerator)
          • Shipping Container (ShippingContainerGenerator)
          • SIN (SINGenerator)
          • SSN (SsnGenerator)
          • Struct Mask (StructMaskGenerator)
          • Timestamp Shift (TimestampShiftGenerator)
          • Unique Email (UniqueEmailGenerator)
          • URL (UrlGenerator)
          • UUID Key (UuidPkGenerator)
          • XML Mask (XmlMaskGenerator)
      • Configure subsetting
      • Check for and resolve schema changes
      • Run data generation jobs
      • Schedule data generation jobs
    • Example script: Starting a data generation job
    • Example script: Polling for a job status and creating a Docker package
Powered by GitBook
On this page

Was this helpful?

Export as PDF
  1. Using the Structural API

Example script: Starting a data generation job

Kicking off a data generation using Python. Note the version that this script was tested against.

tonic_generation.py
# Note that we do not guarantee backwards compatibility for our API.
# Please look at the Swagger API documentation that comes with your instance
# for your exact version's endpoints and parameters. If you are using hosted
# Tonic, you can find the API documentation here:

# Tested 2021.04.12 with Tonic API v199
import json

# Be sure to install requests via pip, pipenv, or poetry before running.
import requests

# Parameters
TONIC_API_KEY = "<<TONIC API TOKEN>>"
TONIC_URL = "https://<<TONIC HOSTNAME>>"
WORKSPACE_NAME = "<<TONIC WORKSPACE>>"


class TonicSession:
    def __init__(self, base_url, api_key):
        self._base_url = base_url
        self._session = requests.Session()
        self._api_key = api_key
        self._session.headers.update({"Authorization": "Apikey {}".format(api_key)})

    def _get_url(self, api_snippet):
        return "{}{}".format(self._base_url, api_snippet)

    def generate_data(self, workspace_id):
        generate_data_url = self._get_url("/api/generateData/start")
        params = {"workspaceId": workspace_id}

        r = self._session.post(generate_data_url, params=params)

        if r.ok:
            print("Data generation started")
        else:
            r.raise_for_status()

    def generate_data_status(self, workspace_id):
        generate_data_status_url = self._get_url("/api/generateData")
        params = {"workspaceId": workspace_id}

        r = self._session.get(generate_data_status_url, params=params)

        if r.ok:
            print(json.dumps(r.json(), indent=2))
        else:
            r.raise_for_status()

    def get_workspaces(self):
        workspace_list_url = self._get_url("/api/workspace")

        r = self._session.get(workspace_list_url)

        if r.ok:
            return r.json()
        else:
            r.raise_for_status()


def find_workspace_id(workspaces_json, workspace_name):
    for workspace in workspaces_json:
        if workspace["workspaceName"] == workspace_name:
            return workspace["id"]
    raise RuntimeError("No Workspace found with name: {}".format(workspace_name))


def main():
    session = TonicSession(TONIC_URL, TONIC_API_KEY)
    workspace_id = find_workspace_id(session.get_workspaces(), WORKSPACE_NAME)

    # Starts a new data generation
    session.generate_data(workspace_id)

    # Prints out the current status of the workspace
    session.generate_data_status(workspace_id)


if __name__ == "__main__":
    main()
tonic_generation.js
// Note that we do not guarantee API back compatibility. Please look at the
// Swagger API documentation that comes with your instance for your exact
// version's endpoints and parameters. If you are using hosted Tonic, you can
// find the API documentation here: https://app.tonic.ai/apidocs/index.html

// Tested 2020.11.30 with Tonic API v145

// Install node-fetch with npm install node-fetch
const fetch = require('node-fetch');

const apiKey = "<<TONIC API TOKEN>>";
const workspaceName = "<<TONIC WORKSPACE>>";
const tonicUrl = "https://<<TONIC HOSTNAME>>"

async function getWorkspaceId() {
    let response = await fetch(`${tonicUrl}/api/workspace`, {
        method: 'get',
        headers: { 'Authorization': `apikey ${apiKey}` },
    });

    const workspaces = await response.json();
    const filteredWorkspaces = workspaces.filter(w => w.fingerprintName === workspaceName);
    if (filteredWorkspaces.length !== 1) {
        throw new Error(`No Workspace found with name: ${workspaceName}`);
    }
    return filteredWorkspaces[0].id;
}

async function generateData(workspaceId) {
    let response = await fetch(`${tonicUrl}/api/generateData/start?workspaceId=${workspaceId}&strictMode=RejectOnAllSchemaItems`, {
        method: 'post',
        headers: { 'Authorization': `apikey ${apiKey}` },
    });

    if(!response.ok) {
        throw new Error(await response.text());
    }

    const job = await response.json();

    return job.id;
}

getWorkspaceId()
.then(workspaceId =>generateData(workspaceId))
.then(jobId => console.log(`Job ${jobId} queued`));
tonic_generation.ps1
# Note that we do not guarantee API back compatibility. Please look at the
# Swagger API documentation that comes with your instance for your exact
# version's endpoints and parameters. If you are using hosted Tonic, you can
# find the API documentation here: https://app.tonic.ai/apidocs/index.html

# Tested 2020.11.30 with Tonic API v145

$apiKey = "<<TONIC API TOKEN>>"
$workspaceName = "<<TONIC WORKSPACE>>"
$tonicUrl = "https://<<TONIC HOSTNAME>>"


$headers = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
$headers.Add("Authorization", "apikey $apiKey")
$headers.Add("Accept", "application/json")

$workspaceResponse = Invoke-RestMethod "$tonicUrl/api/workspace" -Method 'GET' -Headers $headers -UseBasicParsing

$filteredWorkspaces = $workspaceResponse.Where({$_.fingerPrintName -eq $workspaceName})

if ($filteredWorkspaces.Count -ne 1)
{
    throw "No Workspace found with name: $workspaceName"
}

$workspaceId = $filteredWorkspaces[0].id

try {
    $generateResponse = Invoke-RestMethod "$tonicUrl/api/generateData/start?workspaceId=$workspaceId&strictMode=RejectOnAllSchemaItems" -Method 'POST' -Headers $headers -UseBasicParsing
    $jobId = $generateResponse.id
    Write-Host "Job $jobId queued"
}
catch {
    $result = $_.Exception.Response.GetResponseStream()
    $reader = New-Object System.IO.StreamReader($result)
    $reader.BaseStream.Position = 0
    $reader.DiscardBufferedData()
    $responseBody = $reader.ReadToEnd();
    throw $responseBody
}
tonic_generation.sh
#!/bin/bash
# Note that we do not guarantee API back compatibility. Please look at the
# Swagger API documentation that comes with your instance for your exact
# version's endpoints and parameters. If you are using hosted Tonic, you can
# find the API documentation here: https://app.tonic.ai/apidocs/index.html

# Tested 2020.11.30 with Tonic API v145

# This script requires jq to be installed

apikey="<<TONIC API TOKEN>>"
workspaceName="<<TONIC WORKSPACE>>"
tonicUrl="https://<<TONIC HOSTNAME>>"

workspaceResponse=$(curl -v --request GET "${tonicUrl}/api/workspace" -H "Authorization: apikey $apikey" -H "Accept: application/json")

workspaceId=jq '.[] | select(.fingerprintName == $workspaceName) | .id'
if [ ${#workspaceId} != 36 ]
then
    echo "No Workspace found with name: $workspaceName"
    exit 1
fi

generateResponse=$(curl -v --request POST "${tonicUrl}/api/generateData/start?workspaceId=${workspaceId}&strictMode=RejectOnAllSchemaItems" -H "Authorization: apikey $apikey" -H "Accept: application/json" -H "Content-Length: 0")
if [[ $generateResponse == *{* ]]
then
    jobId=$(echo $generateResponse | jq '.id' | tr -d '"')
    echo "Job $jobId queued"
else
    echo $generateResponse | tr -d '"'
    exit 1
fi

Last updated 3 months ago

Was this helpful?