> ## Documentation Index
> Fetch the complete documentation index at: https://private-7c7dfe99-fix-nav-issues.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.
# List ClickPipes
> **This endpoint is in beta.** API contract is stable, and no breaking changes are expected in the future.
Returns a list of ClickPipes.
## OpenAPI
````yaml /_specs/cloud-openapi.json get /v1/organizations/{organizationId}/services/{serviceId}/clickpipes
openapi: 3.0.1
info:
title: OpenAPI spec for ClickHouse Cloud
version: '1.0'
contact:
name: ClickHouse Support
url: >-
https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-299828
email: support@clickhouse.com
servers:
- url: https://api.clickhouse.cloud
security:
- basicAuth: []
tags:
- name: Organization
- name: User management
- name: Billing
- name: Service
- name: Backup
- name: OpenAPI
- name: Prometheus
- name: ClickPipes
paths:
/v1/organizations/{organizationId}/services/{serviceId}/clickpipes:
get:
tags:
- ClickPipes
summary: List ClickPipes
description: >-
**This endpoint is in beta.** API contract is stable, and no breaking
changes are expected in the future.
Returns a list of
ClickPipes.
parameters:
- in: path
name: organizationId
description: ID of the organization that owns the service.
required: true
schema:
type: string
format: uuid
- in: path
name: serviceId
description: ID of the service that owns the ClickPipe.
required: true
schema:
type: string
format: uuid
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
status:
type: number
description: HTTP status code.
example: 200
requestId:
type: string
description: Unique id assigned to every request. UUIDv4
format: uuid
result:
type: array
items:
$ref: '#/components/schemas/ClickPipe'
'400':
description: >-
The server cannot or will not process the request due to something
that is perceived to be a client error.
content:
application/json:
schema:
type: object
properties:
status:
type: number
description: HTTP status code.
example: 400
error:
type: string
description: Detailed error description.
components:
schemas:
ClickPipe:
properties:
id:
description: Unique ClickPipe ID.
type: string
format: uuid
serviceId:
description: ID of the service this ClickPipe belongs to.
type: string
format: uuid
name:
description: Name of the ClickPipe.
type: string
state:
description: Current state of the ClickPipe.
type: string
scaling:
$ref: '#/components/schemas/ClickPipeScaling'
source:
$ref: '#/components/schemas/ClickPipeSource'
destination:
$ref: '#/components/schemas/ClickPipeDestination'
fieldMappings:
type: array
description: >-
Field mappings of the ClickPipe. Note that all destination columns
must be included in the mappings.
items:
$ref: '#/components/schemas/ClickPipeFieldMapping'
settings:
$ref: '#/components/schemas/ClickPipeSettings'
createdAt:
description: Creation date of the ClickPipe.
type: string
updatedAt:
description: Last update date of the ClickPipe.
type: string
ClickPipeScaling:
properties:
replicas:
description: Desired number of replicas. Only for scalable pipes.
type: integer
concurrency:
description: >-
Desired number of concurrency. Only for S3 pipes. If set to 0,
concurrency is auto-scaled based on the cluster memory.
type: integer
deprecated: true
replicaCpuMillicores:
description: CPU in millicores for each replica. Only for streaming pipes.
type: integer
minimum: 125
maximum: 2000
replicaMemoryGb:
description: Memory in GB for each replica. Only for streaming pipes.
type: number
minimum: 0.5
maximum: 8
ClickPipeSource:
properties:
kafka:
oneOf:
- $ref: '#/components/schemas/ClickPipeKafkaSource'
nullable: true
objectStorage:
oneOf:
- $ref: '#/components/schemas/ClickPipeObjectStorageSource'
nullable: true
kinesis:
oneOf:
- $ref: '#/components/schemas/ClickPipeKinesisSource'
nullable: true
postgres:
oneOf:
- $ref: '#/components/schemas/ClickPipePostgresSource'
nullable: true
ClickPipeDestination:
properties:
database:
description: Destination database.
type: string
table:
description: >-
Destination table. Required field for all pipe types except
Postgres.
type: string
managedTable:
description: >-
Is the table managed by ClickPipes? Required field for all pipe
types except Postgres.
type: boolean
tableDefinition:
$ref: '#/components/schemas/ClickPipeDestinationTableDefinition'
columns:
type: array
description: >-
Columns of the destination table. Required field for all pipe types
except Postgres.
items:
$ref: '#/components/schemas/ClickPipeDestinationColumn'
ClickPipeFieldMapping:
properties:
sourceField:
description: Source field name.
type: string
destinationField:
description: Destination field name.
type: string
ClickPipeSettings:
properties:
streaming_max_insert_wait_ms:
description: >-
Streaming max insert wait time. Configures the max wait period
before inserting data into the ClickHouse.
nullable: true
type: integer
minimum: 500
maximum: 60000
example: 5000
object_storage_concurrency:
description: >-
Object storage concurrency. Number of concurrent file processing
threads
nullable: true
type: integer
minimum: 1
maximum: 35
example: 1
object_storage_polling_interval_ms:
description: >-
Object storage polling interval. Configures the refresh interval for
querying continuous ingest for new object storage data
nullable: true
type: integer
minimum: 100
maximum: 3600000
example: 30000
object_storage_max_insert_bytes:
description: >-
Max insert bytes. Number of bytes to process in a single insert
batch
nullable: true
type: integer
minimum: 10485760
maximum: 53687091200
example: 10737418240
object_storage_max_file_count:
description: >-
Max file count. Maximum number of files to process in a single
insert batch
nullable: true
type: integer
minimum: 1
maximum: 10000
example: 100
clickhouse_max_threads:
description: >-
Max threads. Maximum number of concurrent threads for file
processing
nullable: true
type: integer
minimum: 0
maximum: 64
example: 8
clickhouse_max_insert_threads:
description: Max insert threads. Maximum number of concurrent insert threads
nullable: true
type: integer
minimum: 0
maximum: 16
example: 1
clickhouse_min_insert_block_size_bytes:
description: >-
Min insert block size bytes. Minimum size of data block for insert
(in bytes)
nullable: true
type: integer
minimum: 0
maximum: 10737418240
example: 1073741824
clickhouse_max_download_threads:
description: Max download threads. Maximum number of concurrent download threads
nullable: true
type: integer
minimum: 0
maximum: 32
example: 4
clickhouse_parallel_distributed_insert_select:
description: >-
Parallel distributed insert select. Parallel distributed insert
select setting
nullable: true
type: integer
minimum: 0
maximum: 2
example: 2
object_storage_use_cluster_function:
description: >-
use cluster function. Whether to use ClickHouse cluster function for
distributed processing
nullable: true
type: boolean
example: true
clickhouse_parallel_view_processing:
description: >-
parallel view processing. Whether to enable pushing to attached
views concurrently instead of sequentially
nullable: true
type: boolean
example: false
ClickPipeKafkaSource:
properties:
type:
description: Type of the Kafka source.
type: string
enum:
- kafka
- redpanda
- msk
- confluent
- warpstream
- azureeventhub
- dokafka
format:
description: Format of the Kafka source.
type: string
enum:
- JSONEachRow
- Avro
- AvroConfluent
brokers:
description: Brokers of the Kafka source.
type: string
topics:
description: Topics of the Kafka source.
type: string
consumerGroup:
description: >-
Consumer group of the Kafka source. If not provided
"clickpipes-<>" will be used.
nullable: true
type: string
example: my-clickpipe-consumer-group
authentication:
description: >-
Authentication method of the Kafka source. Supported authentication
methods: kafka: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, MUTUAL_TLS,
msk: SCRAM-SHA-512, IAM_ROLE, IAM_USER, MUTUAL_TLS, confluent:
PLAIN, MUTUAL_TLS, warpstream: PLAIN, azureeventhub: PLAIN,
redpanda: SCRAM-SHA-256, SCRAM-SHA-512, MUTUAL_TLS, dokafka:
SCRAM-SHA-256, MUTUAL_TLS
type: string
enum:
- PLAIN
- SCRAM-SHA-256
- SCRAM-SHA-512
- IAM_ROLE
- IAM_USER
- MUTUAL_TLS
iamRole:
description: >-
IAM role for the Kafka source. Use with IAM role authentication.
Read more in ClickPipes documentation:
https://clickhouse.com/docs/en/integrations/clickpipes/kafka#iam
nullable: true
type: string
example: arn:aws:iam::123456789012:role/MyRole
offset:
oneOf:
- $ref: '#/components/schemas/ClickPipeKafkaOffset'
nullable: true
schemaRegistry:
oneOf:
- $ref: '#/components/schemas/ClickPipeKafkaSchemaRegistry'
nullable: true
caCertificate:
description: PEM encoded CA certificates to validate the broker's certificate.
nullable: true
type: string
reversePrivateEndpointIds:
type: array
description: >-
Reverse private endpoint UUIDs used for a secure private connection
to the Kafka source.
items:
type: string
ClickPipeObjectStorageSource:
properties:
type:
description: Type of the ObjectStorage source.
type: string
enum:
- s3
- gcs
- dospaces
- azureblobstorage
format:
description: Format of the files.
type: string
enum:
- JSONEachRow
- JSONAsObject
- CSV
- CSVWithNames
- Parquet
- Avro
url:
description: >-
Provide a path to the file(s) you want to ingest. You can specify
multiple files using bash-like wildcards. For more information, see
the documentation on using wildcards in path:
https://clickhouse.com/docs/en/integrations/clickpipes/object-storage#limitations
type: string
example: >-
https://datasets-documentation.s3.eu-west-3.amazonaws.com/http/**.ndjson.gz
delimiter:
description: Delimiter used in the files.
nullable: true
type: string
example: ','
compression:
description: Compression algorithm used for the files.
nullable: true
type: string
enum:
- gzip
- gz
- brotli
- br
- xz
- LZMA
- zstd
- auto
example: auto
isContinuous:
description: >-
If set to true, the pipe will continuously read new files from the
source. If set to false, the pipe will read the files only once. New
files have to be uploaded lexically order.
nullable: true
type: boolean
queueUrl:
description: >-
SQS queue URL for event-based continuous ingestion. When provided,
files are ingested based on S3 event notifications rather than
lexicographical order. Only applicable when isContinuous is true and
authentication is not public.
nullable: true
type: string
example: https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue
authentication:
description: >-
Authentication method. CONNECTION_STRING is for Azure Blob Storage.
IAM_ROLE and IAM_USER are for AWS S3/GCS/DigitalOcean. If not
provided, no authentication is used.
nullable: true
type: string
enum:
- IAM_ROLE
- IAM_USER
- CONNECTION_STRING
iamRole:
description: >-
IAM role to be used with IAM role authentication. Read more in
ClickPipes documentation:
https://clickhouse.com/docs/en/integrations/clickpipes/object-storage#authentication
nullable: true
type: string
example: arn:aws:iam::123456789012:role/MyRole
connectionString:
description: >-
Connection string for Azure Blob Storage authentication. Required
when authentication is CONNECTION_STRING.
nullable: true
type: string
example: >-
DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net
path:
description: >-
Path to the file(s) within the Azure container. Used for Azure Blob
Storage sources. You can specify multiple files using bash-like
wildcards. For more information, see the documentation on using
wildcards in path:
https://clickhouse.com/docs/en/integrations/clickpipes/object-storage#limitations
nullable: true
type: string
example: data/logs/*.json
azureContainerName:
description: >-
Container name for Azure Blob Storage. Required when type is
azureblobstorage.
nullable: true
type: string
example: mycontainer
ClickPipeKinesisSource:
properties:
format:
description: Format of the Kinesis stream.
type: string
enum:
- JSONEachRow
- Avro
- AvroConfluent
streamName:
description: Name of the Kinesis stream.
type: string
example: my-stream
region:
description: AWS region of the Kinesis stream.
type: string
example: us-east-1
useEnhancedFanOut:
description: Use enhanced fan-out for the Kinesis stream.
nullable: true
type: boolean
iteratorType:
description: >-
Type of iterator to use when reading from the Kinesis stream. If
AT_TIMESTAMP is used, the timestamp field must be provided.
type: string
enum:
- TRIM_HORIZON
- LATEST
- AT_TIMESTAMP
timestamp:
description: >-
UNIX timestamp to start reading from the Kinesis stream. Required if
iteratorType is AT_TIMESTAMP.
nullable: true
type: integer
example: 1615766400
authentication:
description: Authentication method to use with the Kinesis stream.
type: string
enum:
- IAM_ROLE
- IAM_USER
iamRole:
description: IAM role to use for authentication. Required if IAM_ROLE is used.
nullable: true
type: string
example: arn:aws:iam::123456789012:role/MyRole
ClickPipePostgresSource:
properties:
host:
description: Host of the Postgres instance to connect to.
type: string
port:
description: Port of the Postgres instance to connect to.
type: number
database:
description: Database of the Postgres instance to connect to.
type: string
settings:
$ref: '#/components/schemas/ClickPipePostgresPipeSettings'
tableMappings:
type: array
description: Table mappings for Postgres pipe.
items:
$ref: '#/components/schemas/ClickPipePostgresPipeTableMapping'
ClickPipeDestinationTableDefinition:
properties:
engine:
$ref: '#/components/schemas/ClickPipeDestinationTableEngine'
sortingKey:
type: array
description: Sorting key of the destination table. List of columns.
items:
type: string
partitionBy:
description: Partition key SQL expression.
type: string
primaryKey:
description: Primary key of SQL expression.
type: string
ClickPipeDestinationColumn:
properties:
name:
description: Name of the column.
type: string
type:
description: Type of the column.
type: string
ClickPipeKafkaOffset:
properties:
strategy:
description: Offset strategy.
type: string
enum:
- from_beginning
- from_latest
- from_timestamp
timestamp:
description: >-
A minute precision UTC timestamp to start from. Required for
"from_timestamp" strategy.
nullable: true
type: string
example: 2021-01-01T00:00
ClickPipeKafkaSchemaRegistry:
properties:
url:
description: Schema URL. HTTPS required.
type: string
example: https://psrc-aa00.us-east-2.aws.confluent.cloud/schemas/ids/100004
authentication:
description: Authentication type of the schema registry.
type: string
enum:
- PLAIN
caCertificate:
description: >-
PEM encoded CA certificates to validate the schema registry's
certificate.
nullable: true
type: string
ClickPipePostgresPipeSettings:
properties:
syncIntervalSeconds:
description: Interval in seconds to sync data from Postgres.
type: number
pullBatchSize:
description: Number of rows to pull in each batch.
type: number
publicationName:
description: Publication name to use for replication.
type: string
replicationMode:
description: Replication mode to use for the pipe.
type: string
enum:
- cdc
- snapshot
- cdc_only
replicationSlotName:
description: Replication slot name to use for replication.
type: string
allowNullableColumns:
description: Allow nullable columns in the destination table.
type: boolean
initialLoadParallelism:
description: Number of parallel tables to sync during initial load.
type: number
snapshotNumRowsPerPartition:
description: Number of rows to snapshot per partition.
type: number
snapshotNumberOfParallelTables:
description: Number of parallel tables to snapshot.
type: number
enableFailoverSlots:
description: >-
Enable failover for created replication slot. Requires a replication
slot to NOT be set.
type: boolean
ClickPipePostgresPipeTableMapping:
properties:
sourceSchemaName:
description: Source schema name.
type: string
sourceTable:
description: Source table name.
type: string
targetTable:
description: Target table name.
type: string
excludedColumns:
type: array
description: Columns to exclude from the target table.
items:
type: string
useCustomSortingKey:
description: Whether to use a custom sorting key for the target table.
type: boolean
sortingKeys:
type: array
description: Ordered list of columns to use as sorting key for the target table.
items:
type: string
tableEngine:
description: Table engine to use for the target table.
type: string
enum:
- MergeTree
- ReplacingMergeTree
- 'Null'
ClickPipeDestinationTableEngine:
properties:
type:
description: Engine type of the destination table.
type: string
enum:
- MergeTree
- ReplacingMergeTree
- SummingMergeTree
- 'Null'
versionColumnId:
description: Column name to use as version for ReplacingMergeTree engine.
nullable: true
type: string
columnIds:
type: array
description: Column names to sum for SummingMergeTree engine.
items:
type: string
securitySchemes:
basicAuth:
type: http
scheme: basic
description: >-
Use key ID and key secret obtained in ClickHouse Cloud console:
https://clickhouse.com/docs/cloud/manage/openapi
````