diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..94810d00 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +* +!target/*-runner +!target/*-runner.jar +!target/lib/* +!target/quarkus-app/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5b0d9a5d..adc4f6d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,19 @@ +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +release.properties +.flattened-pom.xml + # Eclipse .project -**/.project .classpath -**/.classpath .settings/ -**/.settings/ bin/ -**/bin/ # IntelliJ .idea -**/.idea *.ipr *.iml *.iws @@ -22,6 +25,10 @@ nb-configuration.xml .vscode .factorypath +# OSX +.DS_Store +**/.DS_Store + # Vim *.swp *.swo @@ -30,10 +37,6 @@ nb-configuration.xml *.orig *.rej -# OSX -.DS_Store -**/.DS_Store - # Local environment .env **/.env @@ -42,7 +45,10 @@ nb-configuration.xml *.log /helm/charts/*.tgz - /helm/Chart.lock +# Plugin directory +/.quarkus/cli/plugins/ +# Generated Internal Kind kubeconfig +examples/compose-kind-kafka/kubeconfig \ No newline at end of file diff --git a/.mvn/wrapper/.gitignore b/.mvn/wrapper/.gitignore new file mode 100644 index 00000000..e72f5e8b --- /dev/null +++ b/.mvn/wrapper/.gitignore @@ -0,0 +1 @@ +maven-wrapper.jar diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 00000000..84d1e60d --- /dev/null +++ b/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +public final class MavenWrapperDownloader +{ + private static final String WRAPPER_VERSION = "3.2.0"; + + private static final boolean VERBOSE = Boolean.parseBoolean( System.getenv( "MVNW_VERBOSE" ) ); + + public static void main( String[] args ) + { + log( "Apache Maven Wrapper Downloader " + WRAPPER_VERSION ); + + if ( args.length != 2 ) + { + System.err.println( " - ERROR wrapperUrl or wrapperJarPath parameter missing" ); + System.exit( 1 ); + } + + try + { + log( " - Downloader started" ); + final URL wrapperUrl = new URL( args[0] ); + final String jarPath = args[1].replace( "..", "" ); // Sanitize path + final Path wrapperJarPath = Paths.get( jarPath ).toAbsolutePath().normalize(); + downloadFileFromURL( wrapperUrl, wrapperJarPath ); + log( "Done" ); + } + catch ( IOException e ) + { + System.err.println( "- Error downloading: " + e.getMessage() ); + if ( VERBOSE ) + { + e.printStackTrace(); + } + System.exit( 1 ); + } + } + + private static void downloadFileFromURL( URL wrapperUrl, Path wrapperJarPath ) + throws IOException + { + log( " - Downloading to: " + wrapperJarPath ); + if ( System.getenv( "MVNW_USERNAME" ) != null && System.getenv( "MVNW_PASSWORD" ) != null ) + { + final String username = System.getenv( "MVNW_USERNAME" ); + final char[] password = System.getenv( "MVNW_PASSWORD" ).toCharArray(); + Authenticator.setDefault( new Authenticator() + { + @Override + protected PasswordAuthentication getPasswordAuthentication() + { + return new PasswordAuthentication( username, password ); + } + } ); + } + try ( InputStream inStream = wrapperUrl.openStream() ) + { + Files.copy( inStream, wrapperJarPath, StandardCopyOption.REPLACE_EXISTING ); + } + log( " - Downloader complete" ); + } + + private static void log( String msg ) + { + if ( VERBOSE ) + { + System.out.println( msg ); + } + } + +} diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..346d645f --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/README.md b/README.md index 2993e11f..fffba002 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,85 @@ -# Debezium Management Platform +# Debezium Management Platform -Debezium Management Platform (Debezium Orchestra) aims to provide means to simplify the deployment of -Debezium to various environments in highly opinionated manner. The goal is not to provide -total control over environment specific configuration. To achieve this goal the platform uses -a data-centric view on Debezium components. +> **Note:** This repository has been archived in favour of [debezium-platform](https://github.com/debezium/debezium-platform) + +# debezium-platform-conductor + +The back-end component of Debezium management platform. Conductor provides a set APIs which can be used to orchestrate and control Debezium deployments. Usually it's intended to be interacted with through a front-end client. **Disclaimer**: This project is still in early development stage and should not be used in production. +## Debezium Management Platform + +Debezium Management Platform (Debezium Orchestra) aims to provide means to simplify the deployment of Debezium to various environments in highly opinionated manner. The goal is not to provide total control over environment specific configuration. To achieve this goal the platform uses a data-centric view on Debezium components. + ## Platform Architecture + The platform is composed of two main components: 1. Conductor: The back-end component which provides a set of APIs to orchestrate and control Debezium deployments. 2. Stage: The front-end component which provides a user interface to interact with the Conductor. - ### Conductor Architecture + The conductor component itself is composed of several subcomponents: 1. API Server: The main entry point for the platform. It provides a set of APIs to interact with the platform. -2. Watcher: Component responsible for the actual communication with deployment environment (e.g. Debezium Operator in K8s cluster). +2. Watcher: Component responsible for the actual communication with deployment environment (e.g. Debezium Operator in K8s cluster). - -![Debezium Management Platform Architecture](resources/images/debezium-platform-architecture.svg) +![Debezium Management Platform Architecture](img/debezium-platform-architecture.svg) ## Installation You can install the platform through helm chart. For instructions refer to the [README](helm/README.md) -## Run the example +## How to Try the Platform + +There are currently two examples available to try the platform under `examples` directory: + +1. `compose-rest-api-only`: This example provides a mean to run the API server and the front-end Stage application locally. While data is persisted in a local database, the platform does not interact with any deployment environment (which means no Debezium pipelines are actually deployed). +2. `compose-kind-kafka`: This example uses a more complex environment which relies on local Kubernetes cluster provision via [Kind](https://kind.sigs.k8s.io/) + +### Running the `compose-rest-api-only` Example + +To run the example, execute the following command from the `examples/compose-rest-api-only` directory: + +```shell +# Using podman +podman compose -f compose.yml up + +# Using docker +docker compose -f compose.yml up +``` + +### Running the `compose-kind-kafka` Example -If you don't have already a Kubernetes cluster up, you can use one of the commons tools to have a local K8s cluster +To run the example, execute the following commands from the `examples/compose-kind-kafka` directory: -* [minikube](https://minikube.sigs.k8s.io/docs/) +```shell +# Create the local Kubernetes cluster +./create-cluster.sh + +# Deploy the platform (podman) +docker compose -f compose.yml up + +# Deploy the platform (docker) +docker compose -f compose.yml up +``` + +The `create-cluster.sh` performs several steps: + +1. Creates a local Kubernetes cluster using Kind. +2. Exports "internal" `kubeconfig` file, which is then mounted to the conductor container. +3. Deploys Debezium Operator to the cluster. +4. Deploys Strimzi Operator to the cluster. +5. Provisions PostgreSQL database and Kafka Cluster in the cluster. + +The behaviour of `create-cluster.sh` can be modified by changing the values in the `env.sh` file. + +## Run the example (Helm) + +If you don't have already a Kubernetes cluster up, you can use one of the commons tools to have a local K8s cluster: +* [minikube](https://minikube.sigs.k8s.io/docs/) * [kind](https://kind.sigs.k8s.io/) If you are using `kind` locally, expose ports `80` and `443` on the control-plane node so the ingress can be reached from your host: @@ -42,23 +90,21 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraPortMappings: - - containerPort: 80 - hostPort: 80 - listenAddress: "127.0.0.1" - protocol: TCP - - containerPort: 443 - hostPort: 443 - listenAddress: "127.0.0.1" - protocol: TCP + - containerPort: 80 + hostPort: 80 + listenAddress: "127.0.0.1" + protocol: TCP + - containerPort: 443 + hostPort: 443 + listenAddress: "127.0.0.1" + protocol: TCP ``` ```shell kind create cluster --config kind-ingress.yaml ``` -The prerequisite is to install an ingress controller. - -For example, on `kind` you can install `ingress-nginx` with: +The prerequisite is to install an ingress controller. For example, on `kind` you can install `ingress-nginx` with: ```shell helm upgrade --install ingress-nginx ingress-nginx \ @@ -68,48 +114,41 @@ helm upgrade --install ingress-nginx ingress-nginx \ --set controller.hostPort.enabled=true ``` -For this example, considering a local setup, we will use the `/etc/hosts` to resolve the domain. - -The following script will map the K8s IP to the specified domain into the `/etc/hosts`. +For this example, considering a local setup, we will use the `/etc/hosts` to resolve the domain. The following script will map the K8s IP to the specified domain into the `/etc/hosts`. ```shell export DEBEZIUM_PLATFORM_DOMAIN=platform.debezium.io sudo ./examples/update_hosts.sh ``` -> **_NOTE:_** -If you are using minikube on Mac, you need also to run the `minikube tunnel` command. For more details see [this](https://minikube.sigs.k8s.io/docs/drivers/docker/#known-issues) and [this](https://stackoverflow.com/questions/70961901/ingress-with-minikube-working-differently-on-mac-vs-ubuntu-when-to-set-etc-host). +> **NOTE:** If you are using minikube on Mac, you need also to run the `minikube tunnel` command. For more details see [this](https://minikube.sigs.k8s.io/docs/drivers/docker/#known-issues) and [this](https://stackoverflow.com/questions/70961901/ingress-with-minikube-working-differently-on-mac-vs-ubuntu-when-to-set-etc-host). -> **_NOTE:_** -If you are using Windows, add `127.0.0.1 platform.debezium.io` to `C:\Windows\System32\drivers\etc\hosts`. +> **NOTE:** If you are using Windows, add `127.0.0.1 platform.debezium.io` to `C:\Windows\System32\drivers\etc\hosts`. -Create a dedicated namespace +Create a dedicated namespace: ```shell kubectl create ns debezium-platform ``` -and then install *debezium-platform* through `helm` +and then install *debezium-platform* through `helm`: ```shell -cd helm && -helm dependency build && +cd helm && helm dependency build && helm install debezium-platform . -n debezium-platform -f ../examples/example.yaml && cd .. ``` -after all pods are running you should access the platform UI from `http://platform.debezium.io/` +After all pods are running you should access the platform UI from `http://platform.debezium.io/` -To finish the example we will create a PostgreSQL, that will be used as source database, -and a kafka cluster, used as destination in our example pipeline. +To finish the example we will create a PostgreSQL, that will be used as source database, and a kafka cluster, used as destination in our example pipeline. ```shell # Deploy the source database - kubectl create -n debezium-platform -f examples/k8s/database/001_postgresql.yml ``` -Install the Strimzi operator +Install the Strimzi operator: ```shell helm repo add strimzi https://strimzi.io/charts/ && @@ -119,22 +158,17 @@ helm install strimzi-operator strimzi/strimzi-kafka-operator --version 0.45.1 -- ```shell # Deploy the kafka cluster - kubectl create -n debezium-platform -f examples/k8s/kafka/001_kafka.yml ``` ```shell # Create a test pipeline -# # The script uses the `http` command from HTTPie. - ./examples/seed.sh platform.debezium.io 80 examples/payloads/ ``` -And that's all. - -![Debezium Management Platform Architecture](resources/images/pipeline.png) - -You should have a test pipeline configured to move data from PostgreSQL to Kakfa. +And that's all. +![Debezium Management Platform Pipeline](resources/images/pipeline.png) +You should have a test pipeline configured to move data from PostgreSQL to Kafka. \ No newline at end of file diff --git a/debezium-platform-conductor/src/main/resources/application.yml b/debezium-platform-conductor/src/main/resources/application.yml index 49e30d50..39d0189e 100644 --- a/debezium-platform-conductor/src/main/resources/application.yml +++ b/debezium-platform-conductor/src/main/resources/application.yml @@ -72,7 +72,6 @@ destinations: pulsar: connection: timeout: 60 - rabbitmq: connection: timeout: 60 @@ -124,6 +123,8 @@ quarkus: "registry": host: ${conductor.descriptors.image.registry} enabled: true + username: ${CONDUCTOR_DESCRIPTORS_REGISTRY_USERNAME:} + password: ${CONDUCTOR_DESCRIPTORS_REGISTRY_PASSWORD:} "%dev": conductor: diff --git a/examples/compose-kind-kafka/compose.yml b/examples/compose-kind-kafka/compose.yml new file mode 100644 index 00000000..114f8fe9 --- /dev/null +++ b/examples/compose-kind-kafka/compose.yml @@ -0,0 +1,51 @@ +services: + conductor: + image: quay.io/debezium/platform-conductor:nightly + pull_policy: always + depends_on: + - postgres + ports: + - "8080:8080" + environment: + QUARKUS_DATASOURCE_USERNAME: conductor + QUARKUS_DATASOURCE_PASSWORD: conductor + QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/conductor + CONDUCTOR_WATCHER_ENABLED: true + QUARKUS_KUBERNETES_CLIENT_NAMESPACE: debezium + QUARKUS_HTTP_CORS_ORIGINS: http://localhost:3000 + volumes: + - type: bind + source: ./kubeconfig + target: /home/jboss/.kube/config + networks: + - conductor + - kind + postgres: + image: quay.io/debezium/postgres:14-alpine + ports: + - "5432:5432" + environment: + POSTGRES_USER: conductor + POSTGRES_PASSWORD: conductor + POSTGRES_DB: conductor + networks: + - conductor + stage: + container_name: stage + image: quay.io/debezium/platform-stage:latest + pull_policy: always + depends_on: + - conductor + ports: + - "3000:3000" + environment: + CONDUCTOR_URL: http://localhost:8080 + networks: + - conductor +volumes: + conductor: +networks: + conductor: + external: false + kind: + external: true diff --git a/examples/compose-kind-kafka/create-cluster.sh b/examples/compose-kind-kafka/create-cluster.sh new file mode 100755 index 00000000..d4d466bf --- /dev/null +++ b/examples/compose-kind-kafka/create-cluster.sh @@ -0,0 +1,65 @@ +source env.sh + +echo "### DEBEZIUM DEMO ENVIRONMENT ###" +# Create cluster +echo ">>> Creating Cluster" +kind create cluster --name $CLUSTER +kind get kubeconfig --internal --name $CLUSTER > kubeconfig +kubectl cluster-info --context kind-$CLUSTER + +# Create namespace +echo ">>> Creating namespace" +kubectl create namespace $NAMESPACE +kubectl config set-context --current --namespace $NAMESPACE + +# Install operators +if $DBZ_INSTALL_OPERATOR; then + echo ">>> Installing Debezium operator" + echo ">> Add helm repo" + helm repo add debezium https://charts.debezium.io + helm repo update debezium + echo ">> Deploy operator" + helm install debezium-operator debezium/debezium-operator --version $DBZ_OPERATOR_VERSION --namespace $NAMESPACE +fi + +if $STRIMZI_INSTALL_OPERATOR; then + echo ">>> Installing Strimzi operator" + echo ">> Add helm repo" + helm repo add strimzi https://strimzi.io/charts/ + helm repo update strimzi + echo ">> Deploy operator" + helm install strimzi-operator strimzi/strimzi-kafka-operator --version $STRIMZI_OPERATOR_VERSION --namespace $NAMESPACE +fi + +if $DBZ_INSTALL_OPERATOR; then + kubectl wait --for=condition=Available deployments/debezium-operator --timeout=$TIMEOUT -n $NAMESPACE + echo ">>> Debezium operator ready" +fi + +if $STRIMZI_INSTALL_OPERATOR; then + kubectl wait --for=condition=Available deployments/strimzi-cluster-operator --timeout=$TIMEOUT -n $NAMESPACE + echo ">>> Strimzi operator ready" +fi + +# Deploy Kafka and PostgreSQL +if $STRIMZI_DEPLOY_KAFKA; then + echo ">>> Deploy kafka" + kubectl create -f k8s/kafka/ -n $NAMESPACE +fi + +if $STRIMZI_DEPLOY_KAFKA; then + echo ">> Wait for kafka cluster" + kubectl wait --for=condition=Ready kafkas/dbz-kafka --timeout=$TIMEOUT -n $NAMESPACE + echo ">> Kafka cluster ready" +fi + +if $POSTGRES_DEPLOY; then + echo ">>> Deploy PostgreSQL" + kubectl create -f k8s/database/ -n $NAMESPACE +fi + +if $POSTGRES_DEPLOY; then + echo ">> Wait for PostgreSQL" + kubectl wait --for=condition=Available deployments/postgresql --timeout=$TIMEOUT -n $NAMESPACE + echo ">> PostgreSQL ready" +fi \ No newline at end of file diff --git a/examples/compose-kind-kafka/destroy.sh b/examples/compose-kind-kafka/destroy.sh new file mode 100755 index 00000000..f575aa1d --- /dev/null +++ b/examples/compose-kind-kafka/destroy.sh @@ -0,0 +1,4 @@ +source env.sh + +kind delete cluster --name $CLUSTER +rm -f kubeconfig \ No newline at end of file diff --git a/examples/compose-kind-kafka/env.sh b/examples/compose-kind-kafka/env.sh new file mode 100644 index 00000000..12dc988d --- /dev/null +++ b/examples/compose-kind-kafka/env.sh @@ -0,0 +1,14 @@ +CLUSTER=debezium +NAMESPACE=debezium +TIMEOUT=720s + +DBZ_INSTALL_OPERATOR=true +DBZ_OPERATOR_VERSION=3.0.0-final + +STRIMZI_INSTALL_OPERATOR=true +STRIMZI_OPERATOR_VERSION=0.44.0 +STRIMZI_DEPLOY_KAFKA=true + +POSTGRES_DEPLOY=true + + diff --git a/examples/compose-kind-kafka/k8s/database/001_postgresql.yml b/examples/compose-kind-kafka/k8s/database/001_postgresql.yml new file mode 100644 index 00000000..e44d88d2 --- /dev/null +++ b/examples/compose-kind-kafka/k8s/database/001_postgresql.yml @@ -0,0 +1,73 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +type: opaque +data: + POSTGRES_DB: ZGViZXppdW0= + POSTGRES_USER: ZGViZXppdW0= + POSTGRES_PASSWORD: ZGViZXppdW0= +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: postgresql + labels: + app: postgresql +spec: + replicas: 1 + selector: + matchLabels: + app: postgresql + deployment: postgresql + template: + metadata: + labels: + app: postgresql + deployment: postgresql + spec: + containers: + - resources: {} + name: postgresql + envFrom: + - secretRef: + name: postgresql-credentials + ports: + - containerPort: 5432 + protocol: TCP + imagePullPolicy: IfNotPresent + livenessProbe: + initialDelaySeconds: 30 + tcpSocket: + port: 5432 + timeoutSeconds: 1 + readinessProbe: + exec: + command: + - "/bin/sh" + - "-i" + - "-c" + - "PGPASSWORD=${POSTGRES_PASSWORD} /usr/bin/psql -w -U ${POSTGRES_USER} -d ${POSTGRES_DB} -c 'SELECT 1'" + initialDelaySeconds: 5 + timeoutSeconds: 1 + terminationMessagePolicy: File + terminationMessagePath: /dev/termination-log + image: quay.io/debezium/example-postgres:latest + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: postgresql +spec: + selector: + app: postgresql + deployment: postgresql + ports: + - name: db + port: 5432 + targetPort: 5432 diff --git a/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml b/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml new file mode 100644 index 00000000..6fe30ba9 --- /dev/null +++ b/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml @@ -0,0 +1,48 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaNodePool +metadata: + name: dual-role + labels: + strimzi.io/cluster: dbz-kafka +spec: + replicas: 1 + roles: + - controller + - broker + storage: + type: jbod + volumes: + - id: 0 + type: ephemeral + kraftMetadata: shared +--- + +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: dbz-kafka + annotations: + strimzi.io/node-pools: enabled + strimzi.io/kraft: enabled +spec: + kafka: + version: 3.8.0 + metadataVersion: 3.8-IV0 + listeners: + - name: plain + port: 9092 + type: internal + tls: false + - name: tls + port: 9093 + type: internal + tls: true + config: + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + default.replication.factor: 1 + min.insync.replicas: 1 + entityOperator: + topicOperator: {} + userOperator: {} \ No newline at end of file diff --git a/examples/compose-kind-kafka/payloads/destination.json b/examples/compose-kind-kafka/payloads/destination.json new file mode 100644 index 00000000..475cd1bc --- /dev/null +++ b/examples/compose-kind-kafka/payloads/destination.json @@ -0,0 +1,12 @@ +{ + "name": "test-destination", + "type": "kafka", + "description": "Some funny destination", + "schema": "dummy", + "vaults": [], + "config": { + "producer.bootstrap.servers": "dbz-kafka-kafka-bootstrap.debezium:9092", + "producer.key.serializer": "org.apache.kafka.common.serialization.StringSerializer", + "producer.value.serializer": "org.apache.kafka.common.serialization.StringSerializer" + } +} \ No newline at end of file diff --git a/examples/compose-kind-kafka/payloads/pipeline.json b/examples/compose-kind-kafka/payloads/pipeline.json new file mode 100644 index 00000000..2e30ebfe --- /dev/null +++ b/examples/compose-kind-kafka/payloads/pipeline.json @@ -0,0 +1,13 @@ +{ + "name": "test-pipeline", + "description": "It goes from here to there!", + "source": { + "id": 1, + "name": "test-source" + }, + "destination": { + "id": 1, + "name": "test-destination" + }, + "logLevel": "debug" +} \ No newline at end of file diff --git a/examples/compose-kind-kafka/payloads/source.json b/examples/compose-kind-kafka/payloads/source.json new file mode 100644 index 00000000..aaae4602 --- /dev/null +++ b/examples/compose-kind-kafka/payloads/source.json @@ -0,0 +1,16 @@ +{ + "name": "test-source", + "description": "Yummy data source", + "type": "io.debezium.connector.postgresql.PostgresConnector", + "schema": "dummy", + "vaults": [], + "config": { + "database.hostname": "postgresql", + "database.port": 5432, + "database.user": "debezium", + "database.password": "debezium", + "database.dbname": "debezium", + "topic.prefix": "inventory", + "schema.include.list": "inventory" + } +} \ No newline at end of file diff --git a/examples/compose-kind-kafka/seed.sh b/examples/compose-kind-kafka/seed.sh new file mode 100755 index 00000000..807ae151 --- /dev/null +++ b/examples/compose-kind-kafka/seed.sh @@ -0,0 +1,4 @@ +port=${1:-8080} +http POST localhost:${port}/api/sources @payloads/source.json +http POST localhost:${port}/api/destinations @payloads/destination.json +http POST localhost:${port}/api/pipelines @payloads/pipeline.json \ No newline at end of file diff --git a/examples/compose-rest-api-only/compose.yml b/examples/compose-rest-api-only/compose.yml new file mode 100644 index 00000000..c1a97a5b --- /dev/null +++ b/examples/compose-rest-api-only/compose.yml @@ -0,0 +1,31 @@ +services: + conductor: + image: quay.io/debezium/platform-conductor:nightly + pull_policy: always + depends_on: + - postgres + ports: + - "8080:8080" + environment: + QUARKUS_DATASOURCE_USERNAME: conductor + QUARKUS_DATASOURCE_PASSWORD: conductor + QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/conductor + QUARKUS_HTTP_CORS_ORIGINS: http://localhost:3000 + postgres: + image: quay.io/debezium/postgres:14-alpine + ports: + - "5432:5432" + environment: + POSTGRES_USER: conductor + POSTGRES_PASSWORD: conductor + POSTGRES_DB: conductor + stage: + depends_on: + - conductor + container_name: stage + image: quay.io/debezium/platform-stage:latest + pull_policy: always + ports: + - "3000:3000" + environment: + CONDUCTOR_URL: http://localhost:8080 \ No newline at end of file diff --git a/helm/README.md b/helm/README.md index 08b146c0..b3b4236d 100644 --- a/helm/README.md +++ b/helm/README.md @@ -4,20 +4,66 @@ This chart will install the components required to run the Debezium Platform. 2. Stage: The front-end component which provides a user interface to interact with the Conductor. 3. Debezium operator: operator that manages the creation of Debezium Server custom resource. 4. [Optional] PostgreSQL database used by conductor to store its data. -5. [Optional] Strimzi operator: operator for creating Kakfa cluster. In case you want to use a Kafka destination in you - pipeline. +5. [Optional] Strimzi operator: operator for creating Kafka cluster. In case you want to use a Kafka destination in your pipeline. # Prerequisites -The chart use an ingress to expose `debezium-stage (UI)` and `debezium-conductor (backend)`, -this will require to have -an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) installed in you -cluster. -You need also to have domain that must point to the cluster IP and then configure the `domain.name` property in -you `values.yaml` with your domain. +The chart use an ingress to expose `debezium-stage (UI)` and `debezium-conductor (backend)`, this will require to have an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) installed in your cluster. + +You need also to have a domain that must point to the cluster IP and then configure the `domain.name` property in your `values.yaml` with your domain. ### Configurations +fix-imagepullsecrets-1851 +| Name | Description | Default | +|:---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| +| domain.name | domain used as ingress host | "" | +| domain.url | domain used as ingress host (DEPRECATED). Use `domain.name` instead. | "" | +| ingress.enabled | Enable ingress resource for conductor/stage | true | +| ingress.className | Optional ingress class name | "" | +| ingress.annotations | Extra ingress annotations | {} | +| ingress.tls.enabled | Enable TLS section on ingress | false | +| ingress.tls.secretName | Secret name used when TLS is enabled | "" | +| stage.image | Image for the stage (UI) | quay.io/debezium/platform-stage:latest | +| stage.imagePullPolicy | Image pull policy for the stage container (UI). If empty it will default to IfNotPresent. | IfNotPresent | +| conductor.image | Image for the conductor | quay.io/debezium/platform-conductor:latest | +| conductor.imagePullPolicy | Image pull policy for the conductor container. If empty it will default to IfNotPresent. | IfNotPresent | +| conductor.offset.existingConfigMap | Name of the config map used to store conductor offsets. If empty it will be automatically created. | "" | +| conductor.descriptors.official.enabled | Enable official Debezium descriptors (downloaded via ORAS at startup) | true | +| conductor.descriptors.official.registry | Registry hosting the descriptor OCI artifact | quay.io | +| conductor.descriptors.official.image | Image name for the descriptor OCI artifact | debezium/debezium-descriptors | +| conductor.descriptors.official.tag | Image tag for the descriptor OCI artifact | nightly | +| conductor.descriptors.official.mountPath | Path where descriptors will be downloaded inside the container | /opt/descriptors | +| server.image | Image for Debezium Server instances created by pipelines. If empty, the operator's ServerImageProvider determines the image | "" | +| database.enabled | Enable the installation of PostgreSQL by the chart | false | +| database.name | Database name | postgres | +| database.host | Database host | postgres | +| database.auth.existingSecret | Name of the secret where `username` and `password` are stored. If empty a secret will be created using the `username` and `password` properties | "" | +| database.auth.username | Database username | user | +| database.auth.password | Database password | password | +| debezium-operator.enabled | Enable the installation of the debezium-operator by the chart | true | +| offset.reusePlatformDatabase | Pipelines will use database to store offsets. By default the database used by the conductor service is used. If you want to use a dedicated one set this property to false | true | +| offset.database.name | Database name | postgres | +| offset.database.host | Database host | postgres | +| offset.database.port | Database port | 5432 | +| offset.database.auth.existingSecret | Name of the secret where `username` and `password` are stored. If not set `offset.database.auth.username` and `offset.database.auth.password` will be used. | "" | +| offset.database.auth.username | Database username | user | +| offset.database.auth.password | Database password | password | +| schemaHistory.reusePlatformDatabase | Pipelines will use database to store schema history. By default the database used by the conductor service is used. If you want to use a dedicated one set this property to false | true | +| schemaHistory.database.name | Database name | postgres | +| schemaHistory.database.host | Database host | postgres | +| schemaHistory.database.port | Database port | 5432 | +| schemaHistory.database.auth.existingSecret | Name of the secret where `username` and `password` are stored. If not set `schemaHistory.database.auth.username` and `schemaHistory.database.auth.password` will be used. | "" | +| schemaHistory.database.auth.username | Database username | user | +| schemaHistory.database.auth.password | Database password | password | +| env | List of env variable to pass to the conductor | [] | +| imagePullSecrets | Global list of imagePullSecrets applied to all container images. Per-component `imagePullSecrets` override this value (not merged). | [] | +| stage.imagePullSecrets | Overrides global `imagePullSecrets` for the stage image. | [] | +| conductor.imagePullSecrets | Overrides global `imagePullSecrets` for the conductor image. | [] | +| conductor.descriptors.official.imagePullPolicy | Image pull policy for the descriptors OCI image. | IfNotPresent | +| conductor.descriptors.official.imagePullSecrets | Overrides global `imagePullSecrets` for the descriptors image. | [] | +| server.imagePullPolicy | Image pull policy for Debezium Server instances created by pipelines. | IfNotPresent | +| server.imagePullSecrets | Overrides global `imagePullSecrets` for the Debezium Server image. | [] | | Name | Description | Default | |:-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| | domain.name | domain used as ingress host | "" | @@ -61,10 +107,12 @@ you `values.yaml` with your domain. | schemaHistory.database.auth.password | Database password | password | | | | | env | List of env variable to pass to the conductor | [] | | pipeline.labels | Map of labels to apply to DebeziumServer custom resources created by pipelines. These labels are merged with the internal `debezium.io/conductor-id` label. | {} | +main ## Descriptor OCI Artifacts The conductor uses OCI artifacts containing component descriptors (JSON files) for connectors and transformations. This enables the platform to: + - Discover available connectors and transformations - Render UI forms for configuration - Validate pipeline configurations @@ -72,6 +120,7 @@ The conductor uses OCI artifacts containing component descriptors (JSON files) f ### How it Works At conductor startup, the OCI artifact is downloaded using **ORAS** (OCI Registry as Storage): + - Downloads the descriptor OCI artifact to local filesystem - Extracts all descriptor JSON files - Adds ~5-15 seconds to startup time @@ -99,7 +148,7 @@ conductor: descriptors: official: enabled: true - tag: 3.5.0 # Override just the tag + tag: 3.5.0 ``` To disable descriptors: @@ -114,17 +163,13 @@ conductor: ### OCI Artifact Structure Descriptor OCI artifacts should follow this structure: - -``` debezium-descriptors:nightly -└── / - ├── manifest.json - ├── source-connector/ - │ └── io.debezium.connector.mysql.MySqlConnector.json - └── transformation/ - └── io.debezium.transforms.ExtractNewRecordState.json -``` - +└── / +├── manifest.json +├── source-connector/ +│ └── io.debezium.connector.mysql.MySqlConnector.json +└── transformation/ +└── io.debezium.transforms.ExtractNewRecordState.json The artifact contents are extracted to the configured `mountPath`. ## Debezium Server Image Configuration @@ -141,22 +186,22 @@ server: ### Use Cases - **Pinning to a specific version**: Ensure all pipelines use a specific Debezium Server version - ```yaml +```yaml server: image: quay.io/debezium/server:3.0.0.Final - ``` +``` - **Using custom server image**: Deploy pipelines with a customized Debezium Server image - ```yaml +```yaml server: image: myregistry.io/custom-debezium-server:latest - ``` +``` - **Default behavior**: Leave empty to let the operator decide - ```yaml +```yaml server: - image: "" # Operator's ServerImageProvider determines the image - ``` + image: "" +``` ## Pipeline Labels @@ -181,22 +226,22 @@ The labels are automatically converted to environment variables for the Conducto helm dependency build ``` -Thi will download the required [Debezium Operator](https://github.com/debezium/debezium-operator) chart. +This will download the required [Debezium Operator](https://github.com/debezium/debezium-operator) chart. ```shell -helm install . +helm install . ``` # Uninstall -Find the release name you want to uninstall +Find the release name you want to uninstall: ```shell helm list --all ``` -then uninstall it +then uninstall it: ```shell -helm uninstall -``` \ No newline at end of file +helm uninstall +``` diff --git a/helm/templates/conductor-deployment.yaml b/helm/templates/conductor-deployment.yaml index 5ca9cb85..fdb91c38 100644 --- a/helm/templates/conductor-deployment.yaml +++ b/helm/templates/conductor-deployment.yaml @@ -22,6 +22,13 @@ spec: app.kubernetes.io/name: conductor spec: serviceAccountName: conductor-service-account +{{- $conductorSecrets := .Values.conductor.imagePullSecrets | default list }} +{{- $globalSecrets := .Values.imagePullSecrets | default list }} +{{- $pullSecrets := ternary $conductorSecrets $globalSecrets (gt (len $conductorSecrets) 0) }} +{{- if gt (len $pullSecrets) 0 }} + imagePullSecrets: + {{- toYaml $pullSecrets | nindent 8 }} +{{- end }} containers: - name: conductor image: {{ .Values.conductor.image }} @@ -62,6 +69,18 @@ spec: value: {{ .Values.conductor.descriptors.official.image }} - name: CONDUCTOR_DESCRIPTORS_IMAGE_TAG value: {{ .Values.conductor.descriptors.official.tag }} +{{- if and .Values.conductor.descriptors.official.auth .Values.conductor.descriptors.official.auth.existingSecret }} + - name: CONDUCTOR_DESCRIPTORS_REGISTRY_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.conductor.descriptors.official.auth.existingSecret }} + key: username + - name: CONDUCTOR_DESCRIPTORS_REGISTRY_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.conductor.descriptors.official.auth.existingSecret }} + key: password +{{- end }} {{- end }} {{- if .Values.server.image }} - name: PIPELINE_SERVER_IMAGE diff --git a/helm/templates/stage-deployment.yaml b/helm/templates/stage-deployment.yaml index 41a1c50d..89fde450 100644 --- a/helm/templates/stage-deployment.yaml +++ b/helm/templates/stage-deployment.yaml @@ -19,6 +19,11 @@ spec: labels: app.kubernetes.io/name: stage spec: +{{- $pullSecrets := coalesce .Values.stage.imagePullSecrets .Values.imagePullSecrets | default list }} +{{- if gt (len $pullSecrets) 0 }} + imagePullSecrets: + {{- toYaml $pullSecrets | nindent 8 }} +{{- end }} containers: - image: {{ .Values.stage.image }} imagePullPolicy: {{ .Values.stage.imagePullPolicy }} @@ -29,4 +34,4 @@ spec: env: - name: CONDUCTOR_URL value: {{ required "A valid domain URL required!" (include "debezium-platform.domainUrl" .) }} - restartPolicy: Always + restartPolicy: Always \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 3efe3401..e68af442 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,12 +1,25 @@ domain: url: "" name: "" + +# Global imagePullSecrets applied to all container images by default. +# Per-component imagePullSecrets override this value (not merged). +# Example: +# imagePullSecrets: +# - name: my-registry-secret +imagePullSecrets: [] + stage: image: quay.io/debezium/platform-stage:nightly imagePullPolicy: IfNotPresent + # Overrides global imagePullSecrets for the stage image. + imagePullSecrets: [] + conductor: image: quay.io/debezium/platform-conductor:nightly imagePullPolicy: IfNotPresent + # Overrides global imagePullSecrets for the conductor image. + imagePullSecrets: [] offset: existingConfigMap: "" descriptors: @@ -16,8 +29,17 @@ conductor: image: debezium/debezium-descriptors tag: nightly mountPath: /opt/descriptors + # imagePullPolicy removed - not applicable for ORAS-based pulls + # imagePullSecrets replaced by auth.existingSecret for ORAS credentials + auth: + existingSecret: "" # Secret must contain keys: username, password + server: image: "" + # imagePullPolicy is not yet supported by DebeziumOperator. + # Tracked in: https://github.com/debezium/debezium/issues/1851 + imagePullSecrets: [] + ingress: enabled: true className: "" diff --git a/img/debezium-platform-architecture.svg b/img/debezium-platform-architecture.svg new file mode 100644 index 00000000..e2de7ab5 --- /dev/null +++ b/img/debezium-platform-architecture.svg @@ -0,0 +1,8 @@ + + + + + + + + CONDUCTOR-WATCHERDBZ EngineREST APIENVCONTROLLERDBZ OPERATORMongo_PubSub_PipelinePostgres_Kafka_PipelineCRMSGKUBERNETESSECRETOutboxMSGC1C2C3CONDUCTOR-APISERVICESUSERNext Back+? Source? SinkExtract New StateTransform DatePipeline CompositionCONDUCTORSTAGEUI \ No newline at end of file diff --git a/mvnw b/mvnw new file mode 100755 index 00000000..8d937f4c --- /dev/null +++ b/mvnw @@ -0,0 +1,308 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.2.0 +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "$(uname)" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME + else + JAVA_HOME="/Library/Java/Home"; export JAVA_HOME + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && + JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then + if $darwin ; then + javaHome="$(dirname "\"$javaExecutable\"")" + javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" + else + javaExecutable="$(readlink -f "\"$javaExecutable\"")" + fi + javaHome="$(dirname "\"$javaExecutable\"")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$(cd "$wdir/.." || exit 1; pwd) + fi + # end of workaround + done + printf '%s' "$(cd "$basedir" || exit 1; pwd)" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + # Remove \r in case we run on Windows within Git Bash + # and check out the repository with auto CRLF management + # enabled. Otherwise, we may read lines that are delimited with + # \r\n and produce $'-Xarg\r' rather than -Xarg due to word + # splitting rules. + tr -s '\r\n' ' ' < "$1" + fi +} + +log() { + if [ "$MVNW_VERBOSE" = true ]; then + printf '%s\n' "$1" + fi +} + +BASE_DIR=$(find_maven_basedir "$(dirname "$0")") +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR +log "$MAVEN_PROJECTBASEDIR" + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" +if [ -r "$wrapperJarPath" ]; then + log "Found $wrapperJarPath" +else + log "Couldn't find $wrapperJarPath, downloading it ..." + + if [ -n "$MVNW_REPOURL" ]; then + wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + else + wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + fi + while IFS="=" read -r key value; do + # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) + safeValue=$(echo "$value" | tr -d '\r') + case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; + esac + done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" + log "Downloading from: $wrapperUrl" + + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget > /dev/null; then + log "Found wget ... using wget" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + log "Found curl ... using curl" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + else + curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + fi + else + log "Falling back to using Java to download" + javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" + javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaSource=$(cygpath --path --windows "$javaSource") + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaSource" ]; then + if [ ! -e "$javaClass" ]; then + log " - Compiling MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/javac" "$javaSource") + fi + if [ -e "$javaClass" ]; then + log " - Running MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +# If specified, validate the SHA-256 sum of the Maven wrapper jar file +wrapperSha256Sum="" +while IFS="=" read -r key value; do + case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; + esac +done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" +if [ -n "$wrapperSha256Sum" ]; then + wrapperSha256Result=false + if command -v sha256sum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." + echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + exit 1 + fi + if [ $wrapperSha256Result = false ]; then + echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 + echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 + echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + exit 1 + fi +fi + +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# shellcheck disable=SC2086 # safe args +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 00000000..c4586b56 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,205 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.2.0 +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* +if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %WRAPPER_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file +SET WRAPPER_SHA_256_SUM="" +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B +) +IF NOT %WRAPPER_SHA_256_SUM%=="" ( + powershell -Command "&{"^ + "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ + "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ + " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ + " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ + " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ + " exit 1;"^ + "}"^ + "}" + if ERRORLEVEL 1 goto error +) + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% ^ + %JVM_CONFIG_MAVEN_PROPS% ^ + %MAVEN_OPTS% ^ + %MAVEN_DEBUG_OPTS% ^ + -classpath %WRAPPER_JAR% ^ + "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ + %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" +if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%"=="on" pause + +if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% + +cmd /C exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..af27da93 --- /dev/null +++ b/pom.xml @@ -0,0 +1,228 @@ + + + 4.0.0 + io.debezium + debezium-platform-conductor + 0.1.0-SNAPSHOT + + 3.12.1 + 21 + UTF-8 + UTF-8 + quarkus-bom + io.quarkus.platform + 3.11.2 + true + 3.2.5 + 3.24.2 + 1.18.32 + 1.5.5.Final + + quay.io + debezium + platform-conductor + nightly + linux/amd64,linux/arm64/v8 + + + + + ${quarkus.platform.group-id} + quarkus-bom + ${quarkus.platform.version} + pom + import + + + ${quarkus.platform.group-id} + quarkus-blaze-persistence-bom + ${quarkus.platform.version} + pom + import + + + ${quarkus.platform.group-id} + quarkus-debezium-bom + ${quarkus.platform.version} + pom + import + + + + + + io.debezium + debezium-quarkus-outbox + + + io.debezium + debezium-api + + + io.debezium + debezium-embedded + + + io.debezium + debezium-connector-postgres + + + io.debezium + debezium-operator-api + 3.0.0-SNAPSHOT + + + io.quarkus + quarkus-kubernetes-client + + + io.quarkus + quarkus-config-yaml + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-resteasy-reactive-jackson + + + io.quarkus + quarkus-hibernate-orm + + + io.quarkus + quarkus-hibernate-validator + + + io.quarkus + quarkus-jdbc-postgresql + + + io.quarkus + quarkus-smallrye-openapi + + + io.quarkus + quarkus-websockets-next + + + com.blazebit + blaze-persistence-integration-quarkus-3 + + + com.blazebit + blaze-persistence-integration-hibernate-6.2 + runtime + + + com.blazebit + blaze-persistence-integration-jackson-jakarta + 1.6.11 + + + com.blazebit + blaze-persistence-integration-jaxrs-jackson-jakarta + 1.6.11 + + + org.projectlombok + lombok + ${version.lombok} + provided + + + io.quarkus + quarkus-container-image-jib + + + io.quarkus + quarkus-junit5 + test + + + org.assertj + assertj-core + ${version.assertj} + test + + + org.mapstruct + mapstruct + ${version.mapstruct} + + + + + + ${quarkus.platform.group-id} + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + + build + generate-code + generate-code-tests + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + + -parameters + + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + maven-failsafe-plugin + ${surefire-plugin.version} + + + + integration-test + verify + + + + + + ${project.build.directory}/${project.build.finalName}-runner + org.jboss.logmanager.LogManager + ${maven.home} + + + + + + + + native + + + native + + + + false + native + + + + diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm new file mode 100644 index 00000000..5444af0a --- /dev/null +++ b/src/main/docker/Dockerfile.jvm @@ -0,0 +1,97 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./mvnw package +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/debezium-platform-conductor-jvm . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-jvm +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-jvm +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM registry.access.redhat.com/ubi8/openjdk-21:1.18 + +ENV LANGUAGE='en_US:en' + + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --chown=185 target/quarkus-app/lib/ /deployments/lib/ +COPY --chown=185 target/quarkus-app/*.jar /deployments/ +COPY --chown=185 target/quarkus-app/app/ /deployments/app/ +COPY --chown=185 target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +USER 185 +ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] + diff --git a/src/main/docker/Dockerfile.legacy-jar b/src/main/docker/Dockerfile.legacy-jar new file mode 100644 index 00000000..c952d706 --- /dev/null +++ b/src/main/docker/Dockerfile.legacy-jar @@ -0,0 +1,93 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./mvnw package -Dquarkus.package.type=legacy-jar +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.legacy-jar -t quarkus/debezium-platform-conductor-legacy-jar . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-legacy-jar +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-legacy-jar +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM registry.access.redhat.com/ubi8/openjdk-21:1.18 + +ENV LANGUAGE='en_US:en' + + +COPY target/lib/* /deployments/lib/ +COPY target/*-runner.jar /deployments/quarkus-run.jar + +EXPOSE 8080 +USER 185 +ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] diff --git a/src/main/docker/Dockerfile.native b/src/main/docker/Dockerfile.native new file mode 100644 index 00000000..4785884f --- /dev/null +++ b/src/main/docker/Dockerfile.native @@ -0,0 +1,27 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# +# Before building the container image run: +# +# ./mvnw package -Dnative +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native -t quarkus/debezium-platform-conductor . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor +# +### +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root target/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"] diff --git a/src/main/docker/Dockerfile.native-micro b/src/main/docker/Dockerfile.native-micro new file mode 100644 index 00000000..aa0c26e3 --- /dev/null +++ b/src/main/docker/Dockerfile.native-micro @@ -0,0 +1,30 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# It uses a micro base image, tuned for Quarkus native executables. +# It reduces the size of the resulting container image. +# Check https://quarkus.io/guides/quarkus-runtime-base-image for further information about this image. +# +# Before building the container image run: +# +# ./mvnw package -Dnative +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native-micro -t quarkus/debezium-platform-conductor . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor +# +### +FROM quay.io/quarkus/quarkus-micro-image:2.0 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root target/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"] diff --git a/src/main/java/io/debezium/platform/api/ConductorApiApplication.java b/src/main/java/io/debezium/platform/api/ConductorApiApplication.java new file mode 100644 index 00000000..4e4bf83a --- /dev/null +++ b/src/main/java/io/debezium/platform/api/ConductorApiApplication.java @@ -0,0 +1,8 @@ +package io.debezium.platform.api; + +import jakarta.ws.rs.ApplicationPath; +import jakarta.ws.rs.core.Application; + +@ApplicationPath("/api") +public class ConductorApiApplication extends Application { +} diff --git a/src/main/java/io/debezium/platform/api/DestinationResource.java b/src/main/java/io/debezium/platform/api/DestinationResource.java new file mode 100644 index 00000000..01f8b6a2 --- /dev/null +++ b/src/main/java/io/debezium/platform/api/DestinationResource.java @@ -0,0 +1,110 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.DestinationService; +import io.debezium.platform.domain.views.Destination; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "destinations") +@OpenAPIDefinition( + info = @Info( + title = "Destination API", + description = "CRUD operations over Destination resource", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/destinations") +public class DestinationResource { + + Logger logger; + DestinationService destinationService; + + public DestinationResource(Logger logger, DestinationService destinationService) { + this.logger = logger; + this.destinationService = destinationService; + } + + @Operation(summary = "Returns all available destinations") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Destination.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var destinations = destinationService.list(); + return Response.ok(destinations).build(); + } + + @Operation(summary = "Returns a destination with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Destination.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return destinationService.findById(id) + .map(destination -> Response.ok(destination).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new destination") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Destination destination, @Context UriInfo uriInfo) { + var created = destinationService.create(destination); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing destination") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Destination.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Destination destination) { + var updated = destinationService.update(destination); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing destination") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + destinationService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/api/PipelineLogWebSocket.java b/src/main/java/io/debezium/platform/api/PipelineLogWebSocket.java new file mode 100644 index 00000000..3f3e43e1 --- /dev/null +++ b/src/main/java/io/debezium/platform/api/PipelineLogWebSocket.java @@ -0,0 +1,82 @@ +package io.debezium.platform.api; + + +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.LogStreamingService; +import io.debezium.platform.error.NotFoundException; +import io.quarkus.websockets.next.InboundProcessingMode; +import io.quarkus.websockets.next.OnClose; +import io.quarkus.websockets.next.OnError; +import io.quarkus.websockets.next.OnOpen; +import io.quarkus.websockets.next.PathParam; +import io.quarkus.websockets.next.WebSocket; +import io.quarkus.websockets.next.WebSocketConnection; +import io.smallrye.common.annotation.RunOnVirtualThread; +import jakarta.inject.Inject; +import org.jboss.logging.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@WebSocket( + path = "/api/pipelines/{id}/logs/stream", + inboundProcessingMode = InboundProcessingMode.CONCURRENT +) +public class PipelineLogWebSocket { + + private static final org.slf4j.Logger log = LoggerFactory.getLogger(PipelineLogWebSocket.class); + @Inject + Logger logger; + + @Inject + PipelineService pipelineService; + + @Inject + LogStreamingService logStreamer; + + private final Map streamingTasks = new ConcurrentHashMap<>(); + + + @OnOpen + @RunOnVirtualThread + public void onOpen(@PathParam("id") String idString, WebSocketConnection connection) { + logger.infof("Connection '%s' requesting logs for pipeline '%s',", connection.id(), idString); + var id = Long.parseLong(idString); + + pipelineService + .streamLogs(id, connection::sendTextAndAwait) + .ifPresentOrElse( + task -> streamingTasks.put(connection.id(), task), + () -> { + throw new NotFoundException(id); + }); + } + + @OnError + public void onError(WebSocketConnection connection, @PathParam("id") String idString, NumberFormatException e) { + logger.warnf("Invalid pipeline id: %s", idString); + + connection.sendTextAndAwait("Invalid pipeline id"); + connection.closeAndAwait(); + } + + @OnError + public void onError(WebSocketConnection connection, NotFoundException e) { + logger.warnf("Pipeline not found: %s", e.getId()); + + connection.sendTextAndAwait("Pipeline not found"); + connection.closeAndAwait(); + } + + @OnClose + public void onClose(WebSocketConnection connection) { + logger.debugf("Connection: %s closed", connection.id()); + + var task = streamingTasks.remove(connection.id()); + if (task != null) { + task.stop(); + } + } + +} diff --git a/src/main/java/io/debezium/platform/api/PipelineResource.java b/src/main/java/io/debezium/platform/api/PipelineResource.java new file mode 100644 index 00000000..752c835c --- /dev/null +++ b/src/main/java/io/debezium/platform/api/PipelineResource.java @@ -0,0 +1,135 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.views.Pipeline; +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.logs.LogReader; +import io.smallrye.common.annotation.RunOnVirtualThread; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; + +@Tag(name = "pipelines") +@OpenAPIDefinition( + info = @Info( + title = "Pipeline API", + description = "CRUD operations over Pipeline resource", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/pipelines") +public class PipelineResource { + + Logger logger; + PipelineService pipelineService; + + public PipelineResource(Logger logger, PipelineService pipelineService) { + this.logger = logger; + this.pipelineService = pipelineService; + } + + @Operation(summary = "Returns all available pipelines") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Pipeline.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var pipelines = pipelineService.list(); + return Response.ok(pipelines).build(); + } + + @Operation(summary = "Returns a pipeline with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Pipeline.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return pipelineService.findById(id) + .map(source -> Response.ok(source).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new pipeline") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Pipeline pipeline, @Context UriInfo uriInfo) { + var created = pipelineService.create(pipeline); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing pipeline") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Pipeline.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Pipeline pipeline) { + var updated = pipelineService.update(pipeline); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing pipeline") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + pipelineService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } + + @Operation(summary = "Returns logs for pipeline with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = TEXT_PLAIN, schema = @Schema(implementation = String.class, required = true)) + ) + @GET + @Path("/{id}/logs") + @Produces(TEXT_PLAIN) + @RunOnVirtualThread + public Response getLogById(@PathParam("id") Long id) { + return pipelineService.environmentController(id) + .map(EnvironmentController::pipelines) + .map(pipelines -> pipelines.logReader(id)) + .map(LogReader::readAll) + .map(log -> Response.ok(log) + .header("Content-Disposition", "attachment; filename=pipeline.log") + .build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } +} diff --git a/src/main/java/io/debezium/platform/api/SourceResource.java b/src/main/java/io/debezium/platform/api/SourceResource.java new file mode 100644 index 00000000..4ca1fd61 --- /dev/null +++ b/src/main/java/io/debezium/platform/api/SourceResource.java @@ -0,0 +1,110 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.SourceService; +import io.debezium.platform.domain.views.Source; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "sources") +@OpenAPIDefinition( + info = @Info( + title = "Source API", + description = "CRUD operations over Source resource", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/sources") +public class SourceResource { + + Logger logger; + SourceService sourceService; + + public SourceResource(Logger logger, SourceService sourceService) { + this.logger = logger; + this.sourceService = sourceService; + } + + @Operation(summary = "Returns all available sources") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Source.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var sources = sourceService.list(); + return Response.ok(sources).build(); + } + + @Operation(summary = "Returns a source with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Source.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return sourceService.findById(id) + .map(source -> Response.ok(source).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new source") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Source source, @Context UriInfo uriInfo) { + var created = sourceService.create(source); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing source") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Source.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Source source) { + var updated = sourceService.update(source); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing source") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + sourceService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/api/TransformResource.java b/src/main/java/io/debezium/platform/api/TransformResource.java new file mode 100644 index 00000000..11f9b407 --- /dev/null +++ b/src/main/java/io/debezium/platform/api/TransformResource.java @@ -0,0 +1,110 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.TransformService; +import io.debezium.platform.domain.views.Transform; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "transforms") +@OpenAPIDefinition( + info = @Info( + title = "Transform API", + description = "CRUD operations over Source revault", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/transforms") +public class TransformResource { + + Logger logger; + TransformService transformService; + + public TransformResource(Logger logger, TransformService transformService) { + this.logger = logger; + this.transformService = transformService; + } + + @Operation(summary = "Returns all available vaults") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Transform.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var vaults = transformService.list(); + return Response.ok(vaults).build(); + } + + @Operation(summary = "Returns a transform with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Transform.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return transformService.findById(id) + .map(transform -> Response.ok(transform).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new transform") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Transform transform, @Context UriInfo uriInfo) { + var created = transformService.create(transform); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing transform") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Transform.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Transform transform) { + var updated = transformService.update(transform); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing transform") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + transformService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/api/VaultResource.java b/src/main/java/io/debezium/platform/api/VaultResource.java new file mode 100644 index 00000000..1fa85e2e --- /dev/null +++ b/src/main/java/io/debezium/platform/api/VaultResource.java @@ -0,0 +1,110 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.VaultService; +import io.debezium.platform.domain.views.Vault; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "vaults") +@OpenAPIDefinition( + info = @Info( + title = "Vault API", + description = "CRUD operations over Source revault", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/vaults") +public class VaultResource { + + Logger logger; + VaultService vaultService; + + public VaultResource(Logger logger, VaultService vaultService) { + this.logger = logger; + this.vaultService = vaultService; + } + + @Operation(summary = "Returns all available vaults") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Vault.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var vaults = vaultService.list(); + return Response.ok(vaults).build(); + } + + @Operation(summary = "Returns a vault with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Vault.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return vaultService.findById(id) + .map(vault -> Response.ok(vault).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new vault") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Vault vault, @Context UriInfo uriInfo) { + var created = vaultService.create(vault); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing vault") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Vault.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Vault vault) { + var updated = vaultService.update(vault); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing vault") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + vaultService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/data/model/DestinationEntity.java b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java new file mode 100644 index 00000000..e44b0b54 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java @@ -0,0 +1,42 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Entity(name = "Destination") +@Getter +@Setter +public class DestinationEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + @Column(unique = true, nullable = false) + private String name; + private String description; + @NotEmpty + @Column(nullable = false) + private String type; + @NotEmpty + @Column(nullable = false) + private String schema; + @ManyToMany + @JoinTable(joinColumns = @JoinColumn(name = "destination_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) + private Set vaults = new HashSet<>(); + @JdbcTypeCode(SqlTypes.JSON) + private Map config; +} diff --git a/src/main/java/io/debezium/platform/data/model/PipelineEntity.java b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java new file mode 100644 index 00000000..2ae22721 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java @@ -0,0 +1,42 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.OrderColumn; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; + +import java.util.LinkedList; +import java.util.List; + + +@Entity(name = "Pipeline") +@Getter +@Setter +public class PipelineEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + @Column(unique = true, nullable = false) + private String name; + private String description; + @ManyToOne + private SourceEntity source; + @ManyToOne + private DestinationEntity destination; + @ManyToMany + @JoinTable(joinColumns = @JoinColumn(name = "pipeline_id"), inverseJoinColumns = @JoinColumn(name = "transform_id")) + @OrderColumn + private List transforms = new LinkedList<>(); + @NotEmpty + private String logLevel = "info"; +} + diff --git a/src/main/java/io/debezium/platform/data/model/SourceEntity.java b/src/main/java/io/debezium/platform/data/model/SourceEntity.java new file mode 100644 index 00000000..812b20ed --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/SourceEntity.java @@ -0,0 +1,42 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Entity(name = "Source") +@Getter +@Setter +public class SourceEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + @Column(unique = true, nullable = false) + public String name; + private String description; + @NotEmpty + @Column(nullable = false) + public String type; + @NotEmpty + @Column(nullable = false) + public String schema; + @ManyToMany + @JoinTable(joinColumns = @JoinColumn(name = "source_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) + public Set vaults = new HashSet<>(); + @JdbcTypeCode(SqlTypes.JSON) + public Map config; +} diff --git a/src/main/java/io/debezium/platform/data/model/TransformEntity.java b/src/main/java/io/debezium/platform/data/model/TransformEntity.java new file mode 100644 index 00000000..36bc7fab --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/TransformEntity.java @@ -0,0 +1,42 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Entity(name = "Transform") +@Getter +@Setter +public class TransformEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + @Column(unique = true, nullable = false) + private String name; + private String description; + @NotEmpty + @Column(nullable = false) + private String type; + @NotEmpty + @Column(nullable = false) + private String schema; + @ManyToMany + @JoinTable(joinColumns = @JoinColumn(name = "transform_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) + private Set vaults = new HashSet<>(); + @JdbcTypeCode(SqlTypes.JSON) + private Map config; +} diff --git a/src/main/java/io/debezium/platform/data/model/VaultEntity.java b/src/main/java/io/debezium/platform/data/model/VaultEntity.java new file mode 100644 index 00000000..a02a24b3 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/VaultEntity.java @@ -0,0 +1,30 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.util.HashMap; +import java.util.Map; + +@Entity(name = "Vault") +@Getter +@Setter +public class VaultEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + @Column(unique = true, nullable = false) + private String name; + private String description; + private boolean plaintext = false; + @JdbcTypeCode(SqlTypes.JSON) + private Map items = new HashMap<>(); +} diff --git a/src/main/java/io/debezium/platform/domain/AbstractService.java b/src/main/java/io/debezium/platform/domain/AbstractService.java new file mode 100644 index 00000000..a86be238 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/AbstractService.java @@ -0,0 +1,106 @@ +package io.debezium.platform.domain; + +import com.blazebit.persistence.CriteriaBuilder; +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import com.blazebit.persistence.view.EntityViewSetting; +import io.debezium.platform.domain.views.base.IdView; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import jakarta.validation.Valid; + +import java.util.List; +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.REQUIRED; +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +/** + * Service ancestor + * + * @param EntityType + * @param Updatable View Type + * @param Reference View Type + */ +@Transactional(REQUIRED) +public class AbstractService { + + EntityManager em; + CriteriaBuilderFactory cbf; + EntityViewManager evm; + Class entityType; + Class viewType; + Class referenceViewType; + + public AbstractService( + Class entityType, + Class viewType, + Class referenceViewType, + EntityManager em, + CriteriaBuilderFactory cbf, + EntityViewManager evm) { + this.entityType = entityType; + this.viewType = viewType; + this.referenceViewType = referenceViewType; + this.em = em; + this.cbf = cbf; + this.evm = evm; + } + + public AbstractService() { + // required by CDI + } + + protected CriteriaBuilder cb() { + return cbf.create(em, entityType); + } + + @Transactional(SUPPORTS) + public V viewAs(T view, Class newViewType) { + return evm.convert(view, newViewType); + } + + @Transactional(SUPPORTS) + public List list() { + return evm.applySetting(EntityViewSetting.create(viewType), cb()) + .getResultList(); + } + + @Transactional(SUPPORTS) + public Optional findById(Long id) { + return findByIdAs(viewType, id); + } + + @Transactional(SUPPORTS) + public Optional findByIdAs(Class viewType, Long id) { + var result = evm.find(em, viewType, id); + return Optional.ofNullable(result); + } + + public T create(@Valid T view) { + evm.save(em, view); + onChange(view); + return view; + } + + public T update(@Valid T view) { + evm.save(em, view); + onChange(view); + return view; + } + + public void delete(long id) { + evm.remove(em, referenceViewType, id); + onChange(id); + } + + @Transactional(REQUIRED) + public void onChange(T view) { + // default no-op + } + + @Transactional(REQUIRED) + public void onChange(Long id) { + // default no-op + } +} diff --git a/src/main/java/io/debezium/platform/domain/DestinationService.java b/src/main/java/io/debezium/platform/domain/DestinationService.java new file mode 100644 index 00000000..d2f0133f --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/DestinationService.java @@ -0,0 +1,29 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.data.model.DestinationEntity; +import io.debezium.platform.domain.views.Destination; +import io.debezium.platform.domain.views.refs.DestinationReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +@ApplicationScoped +public class DestinationService extends AbstractService { + + public DestinationService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(DestinationEntity.class, Destination.class, DestinationReference.class, em, cbf, evm); + } + + @Transactional(SUPPORTS) + public Optional findReferenceById(Long id) { + var result = evm.find(em, DestinationReference.class, id); + return Optional.ofNullable(result); + } +} diff --git a/src/main/java/io/debezium/platform/domain/LogStreamingService.java b/src/main/java/io/debezium/platform/domain/LogStreamingService.java new file mode 100644 index 00000000..8ce50a2b --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/LogStreamingService.java @@ -0,0 +1,103 @@ +package io.debezium.platform.domain; + +import io.debezium.platform.environment.logs.LogReader; +import io.quarkus.virtual.threads.VirtualThreads; +import jakarta.enterprise.context.ApplicationScoped; +import lombok.Getter; +import org.jboss.logging.Logger; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Supplier; + +@ApplicationScoped +public class LogStreamingService { + + public static final int NO_DATA_SLEEP_MS = 1000; + + private final Logger logger; + private final ExecutorService executorService; + + public static class LogStreamingTask implements Runnable, Closeable { + private final Logger logger; + + @Getter + private final String name; + private final AtomicBoolean running; + private final Supplier supplier; + private final Consumer consumer; + + public LogStreamingTask(String name, Supplier supplier, Consumer consumer, Logger logger) { + this.name = name; + this.supplier = supplier; + this.consumer = consumer; + this.logger = logger; + this.running = new AtomicBoolean(false); + } + + public boolean isRunning() { + return running.get(); + } + + public void stop() { + if (!running.compareAndSet(true, false)) { + logger.infof("Stopping log streamer for '%s'", name); + } + } + + @Override + public void run() { + if (!running.compareAndSet(false, true)) { + return; + } + logger.infof("Starting log streamer for '%s'", name); + + try (var reader = supplier.get()) { + doStream(reader); + logger.infof("Finished streaming from log %s", name); + } catch (IOException e) { + logger.errorf("Error streaming from log %s", name); + } catch (InterruptedException e) { + logger.errorf("Interrupted while waiting for more logs from log %s", name); + Thread.currentThread().interrupt(); + } finally { + stop(); + } + } + + private void doStream(LogReader reader) throws InterruptedException, IOException { + while (isRunning()) { + var line = reader.readLine(); + if (line == null) { + Thread.sleep(NO_DATA_SLEEP_MS); + } + consumer.accept(line); + } + } + + @Override + public void close() { + stop(); + } + } + + public LogStreamingService(Logger logger, @VirtualThreads ExecutorService executorService) { + this.logger = logger; + this.executorService = executorService; + } + + /** + * Starts streaming the log, passing each line to given consumer. + * + * @param consumer log consumer + */ + public LogStreamingTask stream(String name, Supplier logSupplier, Consumer consumer) { + logger.infof("Starting log streamer for log %s", name); + var task = new LogStreamingTask(name, logSupplier, consumer, logger); + executorService.submit(task); + return task; + } +} diff --git a/src/main/java/io/debezium/platform/domain/PipelineService.java b/src/main/java/io/debezium/platform/domain/PipelineService.java new file mode 100644 index 00000000..d891bb11 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/PipelineService.java @@ -0,0 +1,83 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.outbox.quarkus.ExportedEvent; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.Pipeline; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.domain.views.refs.PipelineReference; +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.watcher.events.PipelineEvent; +import io.quarkus.arc.All; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Event; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; + +@ApplicationScoped +public class PipelineService extends AbstractService { + + private final Event> event; + private final ObjectMapper objectMapper; + private final LogStreamingService logStreamer; + private final List environmentControllers; + + public PipelineService(EntityManager em, + CriteriaBuilderFactory cbf, + EntityViewManager evm, + Event> event, + ObjectMapper objectMapper, + LogStreamingService logStreamer, + @All List environmentControllers) { + super(PipelineEntity.class, Pipeline.class, PipelineReference.class, em, cbf, evm); + this.event = event; + this.objectMapper = objectMapper; + this.logStreamer = logStreamer; + this.environmentControllers = environmentControllers; + } + + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Pipeline view) { + var flat = findByIdAs(PipelineFlat.class, view.getId()).orElseThrow(); + event.fire(PipelineEvent.update(flat, objectMapper)); + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Long id) { + event.fire(PipelineEvent.delete(id)); + } + + /** + * Returns the {@link EnvironmentController} instance for the given pipeline + * + * @param id pipeline id + * @return {@link EnvironmentController} instance for the given pipeline + */ + public Optional environmentController(Long id) { + // TODO: only operator environment is supported currently; + return findById(id).map(pipeline -> environmentControllers.getFirst()); + } + + /** + * Streams logs for the given pipeline, invoking given consumer for each log line + * + * @param id the pipeline id + * @param consumer the consumer to invoke for each log line + * @return log streaming task or empty optional if pipeline was not found + */ + public Optional streamLogs(Long id, Consumer consumer) { + return environmentController(id) + .map(EnvironmentController::pipelines) + .map(pipelines -> logStreamer.stream(String.valueOf(id), () -> pipelines.logReader(id), consumer)); + } +} diff --git a/src/main/java/io/debezium/platform/domain/SourceService.java b/src/main/java/io/debezium/platform/domain/SourceService.java new file mode 100644 index 00000000..4b3ee4fe --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/SourceService.java @@ -0,0 +1,34 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import com.blazebit.persistence.view.EntityViewSetting; +import io.debezium.platform.data.model.SourceEntity; +import io.debezium.platform.domain.views.Source; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.domain.views.refs.SourceReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import jakarta.validation.Valid; + +import java.util.List; +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.REQUIRED; +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +@ApplicationScoped +public class SourceService extends AbstractService { + + public SourceService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(SourceEntity.class, Source.class, SourceReference.class, em, cbf, evm); + } + + @Transactional(SUPPORTS) + public Optional findReferenceById(Long id) { + var result = evm.find(em, SourceReference.class, id); + return Optional.ofNullable(result); + } +} diff --git a/src/main/java/io/debezium/platform/domain/TransformService.java b/src/main/java/io/debezium/platform/domain/TransformService.java new file mode 100644 index 00000000..ec143229 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/TransformService.java @@ -0,0 +1,31 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.data.model.TransformEntity; +import io.debezium.platform.domain.views.Transform; +import io.debezium.platform.domain.views.refs.SourceReference; +import io.debezium.platform.domain.views.refs.TransformReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.REQUIRED; +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +@ApplicationScoped +public class TransformService extends AbstractService { + + public TransformService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(TransformEntity.class, Transform.class, TransformReference.class, em, cbf, evm); + } + + @Transactional(SUPPORTS) + public Optional findReferenceById(Long id) { + var result = evm.find(em, TransformReference.class, id); + return Optional.ofNullable(result); + } +} diff --git a/src/main/java/io/debezium/platform/domain/VaultService.java b/src/main/java/io/debezium/platform/domain/VaultService.java new file mode 100644 index 00000000..73099b2c --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/VaultService.java @@ -0,0 +1,43 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.outbox.quarkus.ExportedEvent; +import io.debezium.platform.data.model.VaultEntity; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.domain.views.refs.VaultReference; +import io.debezium.platform.environment.watcher.events.VaultEvent; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Event; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + + +@ApplicationScoped +public class VaultService extends AbstractService { + + @Inject + Event> event; + + @Inject + ObjectMapper objectMapper; + + public VaultService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(VaultEntity.class, Vault.class, VaultReference.class, em, cbf, evm); + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Vault view) { + event.fire(VaultEvent.update(view, objectMapper)); + } + + @Override + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Long id) { + event.fire(VaultEvent.delete(id)); + } +} diff --git a/src/main/java/io/debezium/platform/domain/views/Destination.java b/src/main/java/io/debezium/platform/domain/views/Destination.java new file mode 100644 index 00000000..0a2cf683 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Destination.java @@ -0,0 +1,12 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.DestinationEntity; + +@EntityView(DestinationEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Destination extends PipelineComponent { +} diff --git a/src/main/java/io/debezium/platform/domain/views/Pipeline.java b/src/main/java/io/debezium/platform/domain/views/Pipeline.java new file mode 100644 index 00000000..da375e58 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Pipeline.java @@ -0,0 +1,35 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.base.NamedView; +import io.debezium.platform.domain.views.refs.DestinationReference; +import io.debezium.platform.domain.views.refs.SourceReference; +import io.debezium.platform.domain.views.refs.TransformReference; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +import java.util.List; + +@EntityView(PipelineEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Pipeline extends NamedView { + String getDescription(); + @NotNull + SourceReference getSource(); + @NotNull + DestinationReference getDestination(); + List getTransforms(); + @NotEmpty + String getLogLevel(); + + void setDescription(String description); + void setName(String name); + void setSource(SourceReference source); + void setDestination(DestinationReference destination); + void setLogLevel(String logLevel); + void setTransforms(List transforms); +} diff --git a/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java new file mode 100644 index 00000000..27ad98fe --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java @@ -0,0 +1,27 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.MappingSingular; +import io.debezium.platform.domain.views.base.NamedView; +import io.debezium.platform.domain.views.refs.VaultReference; +import jakarta.validation.constraints.NotEmpty; + +import java.util.Map; +import java.util.Set; + +public interface PipelineComponent extends NamedView { + String getDescription(); + @NotEmpty + String getType(); + @NotEmpty + String getSchema(); + Set getVaults(); + @MappingSingular + Map getConfig(); + + void setDescription(String description); + void setType(String type); + void setName(String name); + void setSchema(String schema); + void setVaults(Set vaults); + void setConfig(Map config); +} diff --git a/src/main/java/io/debezium/platform/domain/views/Source.java b/src/main/java/io/debezium/platform/domain/views/Source.java new file mode 100644 index 00000000..819adb04 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Source.java @@ -0,0 +1,13 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.SourceEntity; + + +@EntityView(SourceEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Source extends PipelineComponent { +} diff --git a/src/main/java/io/debezium/platform/domain/views/Transform.java b/src/main/java/io/debezium/platform/domain/views/Transform.java new file mode 100644 index 00000000..4dc20b01 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Transform.java @@ -0,0 +1,12 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.TransformEntity; + +@EntityView(TransformEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Transform extends PipelineComponent { +} diff --git a/src/main/java/io/debezium/platform/domain/views/Vault.java b/src/main/java/io/debezium/platform/domain/views/Vault.java new file mode 100644 index 00000000..f6a677ed --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Vault.java @@ -0,0 +1,23 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.MappingSingular; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.VaultEntity; +import io.debezium.platform.domain.views.refs.VaultReference; + +import java.util.Map; + +@EntityView(VaultEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Vault extends VaultReference { + boolean isPlaintext(); + @MappingSingular + Map getItems(); + + void setName(String name); + void setPlaintext(boolean plaintext); + void setItems(Map items); +} diff --git a/src/main/java/io/debezium/platform/domain/views/base/IdView.java b/src/main/java/io/debezium/platform/domain/views/base/IdView.java new file mode 100644 index 00000000..96835bfd --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/base/IdView.java @@ -0,0 +1,8 @@ +package io.debezium.platform.domain.views.base; + +import com.blazebit.persistence.view.IdMapping; + +public interface IdView { + @IdMapping + Long getId(); +} diff --git a/src/main/java/io/debezium/platform/domain/views/base/NamedView.java b/src/main/java/io/debezium/platform/domain/views/base/NamedView.java new file mode 100644 index 00000000..1c69d55b --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/base/NamedView.java @@ -0,0 +1,8 @@ +package io.debezium.platform.domain.views.base; + +import jakarta.validation.constraints.NotEmpty; + +public interface NamedView extends IdView { + @NotEmpty + String getName(); +} diff --git a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java new file mode 100644 index 00000000..85146bb0 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java @@ -0,0 +1,33 @@ +package io.debezium.platform.domain.views.flat; + +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.Destination; +import io.debezium.platform.domain.views.Source; +import io.debezium.platform.domain.views.Transform; +import io.debezium.platform.domain.views.base.NamedView; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +import java.util.List; + +@EntityView(PipelineEntity.class) +@UpdatableEntityView +public interface PipelineFlat extends NamedView { + String getDescription(); + @NotNull + Source getSource(); + @NotNull + Destination getDestination(); + List getTransforms(); + @NotEmpty + String getLogLevel(); + + void setDescription(String description); + void setName(String name); + void setSource(Source source); + void setDestination(Destination destination); + void setLogLevel(String logLevel); + void setTransforms(List transforms); +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/DestinationReference.java b/src/main/java/io/debezium/platform/domain/views/refs/DestinationReference.java new file mode 100644 index 00000000..e0c8a0ff --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/DestinationReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.DestinationEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(DestinationEntity.class) +public interface DestinationReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/PipelineReference.java b/src/main/java/io/debezium/platform/domain/views/refs/PipelineReference.java new file mode 100644 index 00000000..93a7654c --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/PipelineReference.java @@ -0,0 +1,10 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.DestinationEntity; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(PipelineEntity.class) +public interface PipelineReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/SourceReference.java b/src/main/java/io/debezium/platform/domain/views/refs/SourceReference.java new file mode 100644 index 00000000..2217f737 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/SourceReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.SourceEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(SourceEntity.class) +public interface SourceReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/TransformReference.java b/src/main/java/io/debezium/platform/domain/views/refs/TransformReference.java new file mode 100644 index 00000000..83ee9fbe --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/TransformReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.TransformEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(TransformEntity.class) +public interface TransformReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/VaultReference.java b/src/main/java/io/debezium/platform/domain/views/refs/VaultReference.java new file mode 100644 index 00000000..230c141d --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/VaultReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.VaultEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(VaultEntity.class) +public interface VaultReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/environment/EnvironmentController.java b/src/main/java/io/debezium/platform/environment/EnvironmentController.java new file mode 100644 index 00000000..e1d3ab60 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/EnvironmentController.java @@ -0,0 +1,8 @@ +package io.debezium.platform.environment; + +public interface EnvironmentController { + + PipelineController pipelines(); + VaultController vaults(); + +} diff --git a/src/main/java/io/debezium/platform/environment/PipelineController.java b/src/main/java/io/debezium/platform/environment/PipelineController.java new file mode 100644 index 00000000..15a63a9c --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/PipelineController.java @@ -0,0 +1,55 @@ +package io.debezium.platform.environment; + +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.logs.LogReader; + +/** + * Pipeline environment controller + */ +public interface PipelineController { + + /** + * Deploys the pipeline into target environment + *

+ * This method should never be called directly, instead rely on Outbox to + * guarantee the pipeline creation; + *

+ * + * @param pipeline the pipeline to deploy + */ + void deploy(PipelineFlat pipeline); + + /** + * Undeploys the pipeline with given id from target environment + *

+ * This method should never be called directly, instead rely on Outbox to + * guarantee the pipeline removal; + *

+ * + * @param id the pipeline id + */ + void undeploy(Long id); + + /** + * Stops the pipeline with given id + * + * @param id the pipeline id + */ + void stop(Long id); + + /** + * Starts the pipeline with given id + * + * @param id the pipeline id + */ + void start(Long id); + + /** + * Returns the {@link LogReader} instance for the given pipeline + * + * @param id the pipeline id + * @return {@link LogReader} instance for the given pipeline + */ + LogReader logReader(Long id); + +} diff --git a/src/main/java/io/debezium/platform/environment/VaultController.java b/src/main/java/io/debezium/platform/environment/VaultController.java new file mode 100644 index 00000000..6ae46aeb --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/VaultController.java @@ -0,0 +1,10 @@ +package io.debezium.platform.environment; + +import io.debezium.platform.domain.views.Vault; + +public interface VaultController { + + void deploy(Vault vault); + + void undeploy(Long id); +} diff --git a/src/main/java/io/debezium/platform/environment/logs/LogReader.java b/src/main/java/io/debezium/platform/environment/logs/LogReader.java new file mode 100644 index 00000000..0c1e343a --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/logs/LogReader.java @@ -0,0 +1,47 @@ +package io.debezium.platform.environment.logs; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; + +public interface LogReader extends Closeable { + + + /** + * Reads the entire log content currently available. + * + * @return the log content + */ + String readAll(); + + /** + * Obtains a {@link BufferedReader} that can be used to read live logs + *

+ * Note that if this method is called, the reader must be closed by calling {@link #close()}. + *

+ * This method is a shortcut for calling {@link #reader()} and then {@link BufferedReader#readLine()}. + *
+ * Note that if this method is called, the reader must be closed by calling {@link #close()}. + *

+ * + * @return the line read from the log, or {@code null} if the end of the stream has been reached + * @throws IOException if an I/O error occurs + */ + String readLine() throws IOException; + + /** + * Closes the log reader and releases any resources associated + * with it (e.g. {@link BufferedReader} returned by calling {@link #reader()}). + */ + @Override + void close() throws IOException; +} diff --git a/src/main/java/io/debezium/platform/environment/operator/DevClusterInitializer.java b/src/main/java/io/debezium/platform/environment/operator/DevClusterInitializer.java new file mode 100644 index 00000000..86049d61 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/DevClusterInitializer.java @@ -0,0 +1,74 @@ +package io.debezium.platform.environment.operator; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.outbox.quarkus.ExportedEvent; +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.watcher.config.WatcherConfigGroup; +import io.debezium.platform.environment.watcher.events.PipelineEvent; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.quarkus.arc.profile.IfBuildProfile; +import io.quarkus.runtime.Startup; +import io.quarkus.runtime.StartupEvent; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Event; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import org.jboss.logging.Logger; + +import java.net.MalformedURLException; +import java.net.URI; +import java.util.Optional; + +@Startup +@ApplicationScoped +@IfBuildProfile("dev") +public class DevClusterInitializer { + + private final Logger logger; + private final KubernetesClient k8s; + private final WatcherConfigGroup watcherConfig; + private final PipelineService pipelineService; + private final Event> event; + private final ObjectMapper objectMapper; + + @Inject + EntityManager entityManager; + + public DevClusterInitializer(Logger logger, KubernetesClient k8s, + WatcherConfigGroup watcherConfig, PipelineService pipelineService, + Event> event, ObjectMapper objectMapper) { + this.logger = logger; + this.k8s = k8s; + this.watcherConfig = watcherConfig; + this.pipelineService = pipelineService; + this.event = event; + this.objectMapper = objectMapper; + } + + public void init(@Observes StartupEvent event) { + watcherConfig.crd().ifPresent(this::install); + initPipelines(); + } + + public void install(String crdUrl) { + try { + logger.info("Installing CRD from " + crdUrl); + var url = URI.create(crdUrl).toURL(); + var crds = k8s.apiextensions().v1().customResourceDefinitions(); + var crd = crds.load(url).item(); + + crds.resource(crd).serverSideApply(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + + public void initPipelines() { + logger.info("Firing pipeline update events for existing pipelines"); + pipelineService.list().forEach(pipelineService::onChange); + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java new file mode 100644 index 00000000..cc31dd8f --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java @@ -0,0 +1,37 @@ +package io.debezium.platform.environment.operator; + +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.PipelineController; +import io.debezium.platform.environment.VaultController; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Named; +import org.jboss.logging.Logger; + +@ApplicationScoped +@Named(OperatorEnvironmentController.BEAN_NAME) +public class OperatorEnvironmentController implements EnvironmentController { + public static final String BEAN_NAME = "operator-environment-controller"; + + protected final Logger logger; + private final OperatorPipelineController pipelineController; + private final OperatorVaultController vaultController; + + public OperatorEnvironmentController( + Logger logger, + OperatorPipelineController pipelineController, + OperatorVaultController vaultController) { + this.logger = logger; + this.pipelineController = pipelineController; + this.vaultController = vaultController; + } + + @Override + public PipelineController pipelines() { + return pipelineController; + } + + @Override + public VaultController vaults() { + return vaultController; + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java new file mode 100644 index 00000000..f35cda3b --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java @@ -0,0 +1,155 @@ +package io.debezium.platform.environment.operator; + +import io.debezium.operator.api.model.ConfigProperties; +import io.debezium.operator.api.model.DebeziumServer; +import io.debezium.operator.api.model.DebeziumServerBuilder; +import io.debezium.operator.api.model.DebeziumServerSpecBuilder; +import io.debezium.operator.api.model.QuarkusBuilder; +import io.debezium.operator.api.model.SinkBuilder; +import io.debezium.operator.api.model.runtime.RuntimeBuilder; +import io.debezium.operator.api.model.runtime.metrics.JmxExporterBuilder; +import io.debezium.operator.api.model.runtime.metrics.MetricsBuilder; +import io.debezium.operator.api.model.source.OffsetBuilder; +import io.debezium.operator.api.model.source.SchemaHistoryBuilder; +import io.debezium.operator.api.model.source.SourceBuilder; +import io.debezium.operator.api.model.source.storage.offset.InMemoryOffsetStore; +import io.debezium.operator.api.model.source.storage.schema.InMemorySchemaHistoryStore; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.PipelineController; +import io.debezium.platform.environment.operator.logs.KubernetesLogReader; +import io.debezium.platform.environment.logs.LogReader; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.TailPrettyLoggable; +import jakarta.enterprise.context.Dependent; + +import java.util.Map; +import java.util.Optional; + + +@Dependent +public class OperatorPipelineController implements PipelineController { + + public static final String LABEL_DBZ_CONDUCTOR_ID = "debezium.io/conductor-id"; + + private final KubernetesClient k8s; + + public OperatorPipelineController(KubernetesClient k8s) { + this.k8s = k8s; + } + + @Override + public void deploy(PipelineFlat pipeline) { + // Create DS quarkus configuration + var quarkusConfig = new ConfigProperties(); + quarkusConfig.setAllProps(Map.of( + "log.level", pipeline.getLogLevel(), + "log.console.json", false + )); + var dsQuarkus = new QuarkusBuilder() + .withConfig(quarkusConfig) + .build(); + + var dsRuntime = new RuntimeBuilder() + .withMetrics(new MetricsBuilder() + .withJmxExporter(new JmxExporterBuilder() + .withEnabled() + .build()) + .build()) + .build(); + + // Create DS source configuration + var source = pipeline.getSource(); + var sourceConfig = new ConfigProperties(); + sourceConfig.setAllProps(source.getConfig()); + + // TODO: offset and schema history type should be configurable in the future + var offset = new OffsetBuilder() + .withMemory(new InMemoryOffsetStore()) + .build(); + var schemaHistory = new SchemaHistoryBuilder() + .withMemory(new InMemorySchemaHistoryStore()) + .build(); + + var dsSource = new SourceBuilder() + .withSourceClass(source.getType()) + .withOffset(offset) + .withSchemaHistory(schemaHistory) + .withConfig(sourceConfig) + .build(); + + // Create DS sink configuration + var sink = pipeline.getDestination(); + var sinkConfig = new ConfigProperties(); + sinkConfig.setAllProps(sink.getConfig()); + + var dsSink = new SinkBuilder() + .withType(sink.getType()) + .withConfig(sinkConfig) + .build(); + + // Create DS resource + var ds = new DebeziumServerBuilder() + .withMetadata(new ObjectMetaBuilder() + .withName(pipeline.getName()) + .withLabels(Map.of(LABEL_DBZ_CONDUCTOR_ID, pipeline.getId().toString())) + .build()) + .withSpec(new DebeziumServerSpecBuilder() + .withQuarkus(dsQuarkus) + .withRuntime(dsRuntime) + .withSource(dsSource) + .withSink(dsSink) + .build()) + .build(); + + // apply to server + k8s.resource(ds).serverSideApply(); + } + + @Override + public void undeploy(Long id) { + k8s.resources(DebeziumServer.class) + .withLabels(Map.of(LABEL_DBZ_CONDUCTOR_ID, id.toString())) + .delete(); + } + + @Override + public void stop(Long id) { + stop(id, true); + } + + @Override + public void start(Long id) { + stop(id, false); + } + + public Optional findById(Long id) { + return k8s.resources(DebeziumServer.class) + .withLabels(Map.of(LABEL_DBZ_CONDUCTOR_ID, id.toString())) + .list() + .getItems() + .stream() + .findFirst(); + } + + private TailPrettyLoggable findDeploymentLoggable(Long id) { + return findById(id) + .map(DebeziumServer::getMetadata) + .map(ObjectMeta::getName) + .map(name -> k8s.apps().deployments().withName(name)) + .get(); + } + + @Override + public LogReader logReader(Long id) { + return new KubernetesLogReader(() -> findDeploymentLoggable(id)); + } + + private void stop(Long id, boolean stop) { + findById(id).ifPresent(ds -> { + ds.setStopped(stop); + k8s.resource(ds).serverSideApply(); + }); + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorVaultController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorVaultController.java new file mode 100644 index 00000000..83e7f750 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorVaultController.java @@ -0,0 +1,26 @@ +package io.debezium.platform.environment.operator; + +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.environment.VaultController; +import io.fabric8.kubernetes.client.KubernetesClient; +import jakarta.enterprise.context.Dependent; + + +@Dependent +public class OperatorVaultController implements VaultController { + + private final KubernetesClient k8s; + + public OperatorVaultController(KubernetesClient k8s) { + this.k8s = k8s; + } + + @Override + public void deploy(Vault vault) { + } + + @Override + public void undeploy(Long id) { + + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java b/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java new file mode 100644 index 00000000..5a5f56e4 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java @@ -0,0 +1,61 @@ +package io.debezium.platform.environment.operator.logs; + +import io.debezium.platform.environment.logs.LogReader; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.dsl.LogWatch; +import io.fabric8.kubernetes.client.dsl.TailPrettyLoggable; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Objects; +import java.util.function.Supplier; + +public class KubernetesLogReader implements LogReader { + + public static final int STREAM_TAIL_LINES = 100; + + private final Supplier supplier; + private LogWatch watch; + private BufferedReader reader; + + public KubernetesLogReader(Supplier supplier) { + this.supplier = Objects.requireNonNull(supplier, "Supplier cannot be null"); + } + + public String readAll() { + return supplier.get().getLog(); + } + + @Override + public BufferedReader reader() throws IOException { + return ensureReader(); + } + + @Override + public String readLine() throws IOException { + return reader().readLine(); + } + + @Override + public void close() throws IOException { + if (reader != null) { + reader.close(); + } + if (watch != null) { + watch.close(); + } + } + + private BufferedReader ensureReader() throws IOException { + if (reader == null) { + try { + this.watch = supplier.get().tailingLines(STREAM_TAIL_LINES).watchLog(); + this.reader = new BufferedReader(new InputStreamReader(watch.getOutput())); + } catch (KubernetesClientException e) { + throw new IOException(e); + } + } + return reader; + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/ConductorEnvironmentWatcher.java b/src/main/java/io/debezium/platform/environment/watcher/ConductorEnvironmentWatcher.java new file mode 100644 index 00000000..f0be55ed --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/ConductorEnvironmentWatcher.java @@ -0,0 +1,98 @@ +package io.debezium.platform.environment.watcher; + +import io.debezium.config.Configuration; +import io.debezium.connector.postgresql.PostgresConnector; +import io.debezium.connector.postgresql.PostgresConnectorConfig; +import io.debezium.embedded.Connect; +import io.debezium.embedded.EmbeddedEngineConfig; +import io.debezium.engine.DebeziumEngine; +import io.debezium.platform.environment.watcher.config.WatcherConfig; +import io.debezium.platform.environment.watcher.consumers.OutboxParentEventConsumer; +import io.debezium.transforms.outbox.EventRouter; +import io.quarkus.runtime.ShutdownEvent; +import io.quarkus.runtime.Startup; +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import org.jboss.logging.Logger; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@ApplicationScoped +@Startup +public class ConductorEnvironmentWatcher { + + private final Logger logger; + private final OutboxParentEventConsumer eventConsumer; + private final WatcherConfig watcherConfig; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private DebeziumEngine engine; + + public ConductorEnvironmentWatcher(Logger logger, WatcherConfig watcherConfig, OutboxParentEventConsumer eventConsumer) { + this.logger = logger; + this.watcherConfig = watcherConfig; + this.eventConsumer = eventConsumer; + } + + @PostConstruct + public void start() { + if (!watcherConfig.watcher().enabled()) { + logger.info("Skipping watcher because it is not enabled"); + return; + } + + var connection = watcherConfig.connection(); + var offset = watcherConfig.watcher().offset(); + var outbox = watcherConfig.outbox(); + var extraFields = Stream.of(outbox.aggregateColumn(), outbox.aggregateIdColumn(), outbox.typeColumn()) + .map(c -> c + ":envelope") + .collect(Collectors.joining(",")); + + var config = Configuration.create() + .with(EmbeddedEngineConfig.ENGINE_NAME, "conductor") + .with(EmbeddedEngineConfig.CONNECTOR_CLASS, PostgresConnector.class.getName()) + .with(EmbeddedEngineConfig.OFFSET_STORAGE, offset.storage()) + .with(EmbeddedEngineConfig.OFFSET_STORAGE_FILE_FILENAME, offset.file()) + .with(PostgresConnectorConfig.TOPIC_PREFIX, "conductor") + .with(PostgresConnectorConfig.HOSTNAME, connection.host()) + .with(PostgresConnectorConfig.PORT, connection.port()) + .with(PostgresConnectorConfig.USER, connection.username()) + .with(PostgresConnectorConfig.PASSWORD, connection.password()) + .with(PostgresConnectorConfig.DATABASE_NAME, connection.database()) + .with(PostgresConnectorConfig.PLUGIN_NAME, PostgresConnectorConfig.LogicalDecoder.PGOUTPUT.getValue()) + .with(PostgresConnectorConfig.INCLUDE_SCHEMA_CHANGES, false) + .with(PostgresConnectorConfig.TABLE_INCLUDE_LIST, "public.%s".formatted(outbox.table())) + .with("transforms", "outbox") + .with("transforms.outbox.type", EventRouter.class.getName()) + .with("transforms.outbox.table.fields.additional.placement", extraFields) + .build(); + + logger.info("Creating Debezium engine"); + this.engine = DebeziumEngine.create(Connect.class) + .using(config.asProperties()) + .notifying(eventConsumer) + .build(); + + logger.info("Attempting to start debezium engine"); + executor.execute(engine); + } + + public void stop(@Observes ShutdownEvent event) { + if (engine == null) { + return; + } + + try { + logger.info("Attempting to stop Debezium"); + engine.close(); + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } catch (Exception e) { + logger.error("Exception while shutting down Debezium", e); + } + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/DataSourceConfigGroup.java b/src/main/java/io/debezium/platform/environment/watcher/config/DataSourceConfigGroup.java new file mode 100644 index 00000000..136ea0d1 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/DataSourceConfigGroup.java @@ -0,0 +1,13 @@ +package io.debezium.platform.environment.watcher.config; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; + +@ConfigMapping(prefix = "quarkus.datasource") +public interface DataSourceConfigGroup { + + String username(); + String password(); + @WithName("jdbc.url") + String url(); +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/OutboxConfigGroup.java b/src/main/java/io/debezium/platform/environment/watcher/config/OutboxConfigGroup.java new file mode 100644 index 00000000..bb4127ca --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/OutboxConfigGroup.java @@ -0,0 +1,20 @@ +package io.debezium.platform.environment.watcher.config; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; + +@ConfigMapping(prefix = "quarkus.debezium-outbox") +public interface OutboxConfigGroup { + + @WithName("table-name") + String table(); + + @WithName("aggregate-type.name") + String aggregateColumn(); + + @WithName("aggregate-id.name") + String aggregateIdColumn(); + + @WithName("type.name") + String typeColumn(); +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfig.java b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfig.java new file mode 100644 index 00000000..35778845 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfig.java @@ -0,0 +1,41 @@ +package io.debezium.platform.environment.watcher.config; + +import jakarta.enterprise.context.Dependent; +import lombok.Getter; +import lombok.experimental.Accessors; + +@Dependent +@Accessors(fluent = true) +@Getter +public final class WatcherConfig { + + private final WatcherConfigGroup watcher; + private final OutboxConfigGroup outbox; + private final ConnectionConfig connection; + + public record ConnectionConfig(String username, String password, String database, String host, int port) {} + + public WatcherConfig(DataSourceConfigGroup dsConfig, WatcherConfigGroup watcher, OutboxConfigGroup outbox) { + this.watcher = watcher; + this.outbox = outbox; + this.connection = createConnectionConfig(dsConfig); + } + + private static ConnectionConfig createConnectionConfig(DataSourceConfigGroup dsConfig) { + var url = dsConfig.url(); + var hostStart = url.indexOf("://") + 3; + var hostEnd = url.indexOf(":", hostStart); + var portStart = hostEnd + 1; + var portEnd = url.indexOf("/", portStart); + var databaseStart = portEnd + 1; + var databaseEnd = url.indexOf("?", databaseStart); + + var username = dsConfig.username(); + var password = dsConfig.password(); + var database = url.substring(databaseStart, (databaseEnd != -1)? databaseEnd : url.length()); + var host = url.substring(hostStart, hostEnd); + var port = url.substring(portStart, portEnd); + + return new ConnectionConfig(username, password, database, host, Integer.parseInt(port)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfigGroup.java b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfigGroup.java new file mode 100644 index 00000000..82078997 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfigGroup.java @@ -0,0 +1,20 @@ +package io.debezium.platform.environment.watcher.config; + +import io.smallrye.config.ConfigMapping; + +import java.util.Optional; + +@ConfigMapping(prefix = "conductor.watcher") +public interface WatcherConfigGroup { + + boolean enabled(); + + Optional crd(); + + OffsetConfigGroup offset(); + + interface OffsetConfigGroup { + String storage(); + String file(); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/AbstractEventConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/AbstractEventConsumer.java new file mode 100644 index 00000000..31ec8103 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/AbstractEventConsumer.java @@ -0,0 +1,38 @@ +package io.debezium.platform.environment.watcher.consumers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.environment.EnvironmentController; +import org.jboss.logging.Logger; + +public abstract class AbstractEventConsumer implements EnvironmentEventConsumer { + + protected final Logger logger; + protected final EnvironmentController environment; + protected final ObjectMapper objectMapper; + protected final Class payloadType; + + public AbstractEventConsumer(Logger logger, EnvironmentController environment, ObjectMapper objectMapper, Class payloadType) { + this.logger = logger; + this.environment = environment; + this.objectMapper = objectMapper; + this.payloadType = payloadType; + } + + @Override + public Class consumedPayloadType() { + return payloadType; + } + + @Override + public T convert(String payload) { + if (payload == null) { + return null; + } + try { + return objectMapper.readValue(payload, consumedPayloadType()); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/EnvironmentEventConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/EnvironmentEventConsumer.java new file mode 100644 index 00000000..0c4f9700 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/EnvironmentEventConsumer.java @@ -0,0 +1,74 @@ +package io.debezium.platform.environment.watcher.consumers; + +import java.util.Collection; +import java.util.Optional; +import java.util.function.BiConsumer; + + +/** + * Event consumers + * @param payload type + */ +public interface EnvironmentEventConsumer extends BiConsumer> { + + /** + * @return collection of consumable aggregate types + */ + Collection consumedAggregates(); + + /** + * @return collection of consumable event types + */ + Collection consumedTypes(); + + /** + * @return consumed payload class + */ + Class consumedPayloadType(); + + /** + * Determines whether this consumer consumes events for given aggregate + * and event types. By default, empty list returned by + * {@link #consumedAggregates()} or {@link #consumedTypes()} is + * treated as a wildcard. + * + * @param aggregateType event aggregate eventType + * @param eventType event eventType + * @return true if the event should be consumed false otherwise + */ + default boolean consumes(String aggregateType, String eventType) { + var aggregates = consumedAggregates(); + var types = consumedTypes(); + + if (!aggregates.isEmpty() && !aggregates.contains(aggregateType)) { + return false; + } + + return types.isEmpty() || types.contains(eventType); + } + + /** + * Converts json payload into object accepted by this consumer + * @param payload json payload + * + * @return converted payload object + */ + T convert(String payload); + + /** + * A shortcut method which takes event aggregate,type and payload. + * Determines whether this consumer is applicable and if so, converts the payload and + * calls {@link #accept(Object, Object)} )}. + * + * @param aggregateType event aggregate type + * @param eventType event type + * @param id aggregate id + * @param payload json payload + */ + default void consume(String aggregateType, String eventType, Long id, String payload) { + if (consumes(aggregateType, eventType)) { + var object = convert(payload); + accept(id, Optional.ofNullable(object)); + } + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/OutboxParentEventConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/OutboxParentEventConsumer.java new file mode 100644 index 00000000..89926e89 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/OutboxParentEventConsumer.java @@ -0,0 +1,49 @@ +package io.debezium.platform.environment.watcher.consumers; + +import io.debezium.engine.ChangeEvent; +import io.debezium.platform.environment.watcher.config.OutboxConfigGroup; +import jakarta.enterprise.context.Dependent; +import jakarta.enterprise.inject.Instance; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.jboss.logging.Logger; + +import java.util.function.Consumer; + +/** + * Top level consumer of outbox events. Parent consumer will extract + * required information from the accepted {@link ChangeEvent} and delegate + * to all registered instances of {@link EnvironmentEventConsumer} + *
+ * + * It's then up to {@link EnvironmentEventConsumer} instances to either process + * or ignore the event. + */ +@Dependent +public final class OutboxParentEventConsumer implements Consumer> { + + private final Logger logger; + private final OutboxConfigGroup outbox; + private final Instance> eventConsumers; + + public OutboxParentEventConsumer(Logger logger, OutboxConfigGroup outbox, Instance> eventConsumers) { + this.logger = logger; + this.outbox = outbox; + this.eventConsumers = eventConsumers; + } + + @Override + public void accept(ChangeEvent event) { + var value = (Struct) event.value().value(); + + var aggregateType = value.getString(outbox.aggregateColumn()); + var aggregateId = value.getString(outbox.aggregateIdColumn()); + var eventType = value.getString(outbox.typeColumn()); + var payload = value.getString("payload"); + + logger.debugf("Consumed %s event for % (#%d)", eventType, aggregateType, aggregateId); + + eventConsumers.forEach(consumer -> consumer.consume( + aggregateType, eventType, Long.valueOf(aggregateId), payload)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/PipelineConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/PipelineConsumer.java new file mode 100644 index 00000000..1004cf8d --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/PipelineConsumer.java @@ -0,0 +1,39 @@ +package io.debezium.platform.environment.watcher.consumers; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.watcher.events.EventType; +import jakarta.enterprise.context.Dependent; +import org.jboss.logging.Logger; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Dependent +public class PipelineConsumer extends AbstractEventConsumer { + + public PipelineConsumer(Logger logger, EnvironmentController environment, ObjectMapper objectMapper) { + super(logger, environment, objectMapper, PipelineFlat.class); + } + + @Override + public Collection consumedAggregates() { + return List.of("pipeline"); + } + + @Override + public Collection consumedTypes() { + return List.of(EventType.UPDATE.name(), EventType.DELETE.name()); + } + + @Override + public void accept(Long id, Optional payload) { + logger.info("Received pipeline event: " + id); + logger.info(">>> payload: \n" + payload); + var pipelines = environment.pipelines(); + + payload.ifPresentOrElse(pipelines::deploy, () -> pipelines.undeploy(id)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/VaultConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/VaultConsumer.java new file mode 100644 index 00000000..b59d658e --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/VaultConsumer.java @@ -0,0 +1,44 @@ +package io.debezium.platform.environment.watcher.consumers; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.watcher.events.EventType; +import jakarta.enterprise.context.Dependent; +import org.jboss.logging.Logger; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Dependent +public class VaultConsumer extends AbstractEventConsumer { + + public VaultConsumer(Logger logger, EnvironmentController environment, ObjectMapper objectMapper) { + super(logger, environment, objectMapper, Vault.class); + } + + @Override + public Collection consumedAggregates() { + return List.of("vault"); + } + + @Override + public Collection consumedTypes() { + return List.of(EventType.UPDATE.name(), EventType.DELETE.name()); + } + + @Override + public Class consumedPayloadType() { + return Vault.class; + } + + @Override + public void accept(Long id, Optional payload) { + logger.info("Received vault event: " + id); + logger.info(">>> payload: \n" + payload); + var vaults = environment.vaults(); + + payload.ifPresentOrElse(vaults::deploy, () -> vaults.undeploy(id)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/AbstractEvent.java b/src/main/java/io/debezium/platform/environment/watcher/events/AbstractEvent.java new file mode 100644 index 00000000..89d09ccf --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/AbstractEvent.java @@ -0,0 +1,33 @@ +package io.debezium.platform.environment.watcher.events; + +import com.fasterxml.jackson.databind.JsonNode; +import io.debezium.outbox.quarkus.ExportedEvent; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; + +@Getter +@Setter +public abstract class AbstractEvent + implements ExportedEvent { + + private final String aggregateType; + private final String aggregateId; + private final String type; + private final JsonNode payload; + private final Instant timestamp; + + public AbstractEvent( + String aggregateType, + String aggregateId, + EventType type, + Instant timestamp, + JsonNode payload) { + this.aggregateType = aggregateType; + this.aggregateId = aggregateId; + this.type = type.name(); + this.payload = payload; + this.timestamp = timestamp; + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/EventType.java b/src/main/java/io/debezium/platform/environment/watcher/events/EventType.java new file mode 100644 index 00000000..b6228288 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/EventType.java @@ -0,0 +1,6 @@ +package io.debezium.platform.environment.watcher.events; + +public enum EventType { + UPDATE, + DELETE +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/PipelineEvent.java b/src/main/java/io/debezium/platform/environment/watcher/events/PipelineEvent.java new file mode 100644 index 00000000..fa855f0f --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/PipelineEvent.java @@ -0,0 +1,25 @@ +package io.debezium.platform.environment.watcher.events; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.flat.PipelineFlat; + +import java.time.Instant; + +public final class PipelineEvent extends AbstractEvent { + + private static final String AGGREGATE_TYPE = "pipeline"; + + private PipelineEvent(String aggregateId, EventType type, Instant timestamp, JsonNode payload) { + super(AGGREGATE_TYPE, aggregateId, type, timestamp, payload); + } + + public static PipelineEvent update(PipelineFlat pipeline, ObjectMapper objectMapper) { + var payload = objectMapper.valueToTree(pipeline); + return new PipelineEvent(pipeline.getId().toString(), EventType.UPDATE, Instant.now(), payload); + } + + public static PipelineEvent delete(Long id) { + return new PipelineEvent(id.toString(), EventType.DELETE, Instant.now(), null); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/VaultEvent.java b/src/main/java/io/debezium/platform/environment/watcher/events/VaultEvent.java new file mode 100644 index 00000000..753f71f7 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/VaultEvent.java @@ -0,0 +1,25 @@ +package io.debezium.platform.environment.watcher.events; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.Vault; + +import java.time.Instant; + +public final class VaultEvent extends AbstractEvent { + + private static final String AGGREGATE_TYPE = "vault"; + + private VaultEvent(String aggregateId, EventType type, Instant timestamp, JsonNode payload) { + super(AGGREGATE_TYPE, aggregateId, type, timestamp, payload); + } + + public static VaultEvent update(Vault vault, ObjectMapper objectMapper) { + var payload = objectMapper.valueToTree(vault); + return new VaultEvent(vault.getId().toString(), EventType.UPDATE, Instant.now(), payload); + } + + public static VaultEvent delete(Long id) { + return new VaultEvent(id.toString(), EventType.DELETE, Instant.now(), null); + } +} diff --git a/src/main/java/io/debezium/platform/error/NotFoundException.java b/src/main/java/io/debezium/platform/error/NotFoundException.java new file mode 100644 index 00000000..ac5ee3b8 --- /dev/null +++ b/src/main/java/io/debezium/platform/error/NotFoundException.java @@ -0,0 +1,15 @@ +package io.debezium.platform.error; + +public class NotFoundException extends RuntimeException { + + private final long id; + + public NotFoundException(Long id) { + super("Invalid resource with id: " + id); + this.id = id; + } + + public long getId() { + return id; + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 00000000..772093c3 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,51 @@ +conductor: + watcher: +# For now watcher is disabled in prod as we need only the REST API + enabled: false + offset: + storage: org.apache.kafka.connect.storage.FileOffsetBackingStore + file: offsets.dat + +quarkus: + http: + cors: + ~: true + debezium-outbox: + table-name: events + aggregate-type: + name: aggregatetype + aggregate-id: + name: aggregateid + type: + name: type + datasource: + db-kind: postgresql + hibernate-orm: + database: + generation: drop-and-create + swagger-ui: + always-include: true + log: + level: INFO + +"%dev": + conductor: + watcher: + enabled: true + crd: https://raw.githubusercontent.com/debezium/debezium-operator/main/k8/debeziumservers.debezium.io-v1.yml + offset: + storage: org.apache.kafka.connect.storage.MemoryOffsetBackingStore + quarkus: + debezium-outbox: + remove-after-insert: false + datasource: + devservices: + enabled: true + port: 5432 + image-name: quay.io/debezium/postgres:16-alpine + http: + port: 8081 + log: + level: INFO + category: + "io.debezium": DEBUG \ No newline at end of file diff --git a/src/test/java/io/debezium/platform/ServiceTest.java b/src/test/java/io/debezium/platform/ServiceTest.java new file mode 100644 index 00000000..f1e7aae5 --- /dev/null +++ b/src/test/java/io/debezium/platform/ServiceTest.java @@ -0,0 +1,211 @@ +package io.debezium.platform; + +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.domain.DestinationService; +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.SourceService; +import io.debezium.platform.domain.TransformService; +import io.debezium.platform.domain.VaultService; +import io.debezium.platform.domain.views.Destination; +import io.debezium.platform.domain.views.Pipeline; +import io.debezium.platform.domain.views.Source; +import io.debezium.platform.domain.views.Transform; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.domain.views.refs.DestinationReference; +import io.debezium.platform.domain.views.refs.SourceReference; +import io.debezium.platform.domain.views.refs.VaultReference; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import org.assertj.core.api.Assertions; +import org.assertj.core.api.Assumptions; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ServiceTest { + @Inject + SourceService sourceService; + + @Inject + PipelineService pipelineService; + @Inject + DestinationService destinationService; + @Inject + VaultService vaultService; + @Inject + TransformService transformService; + + @Inject + EntityViewManager evm; + + static List sources = new ArrayList<>(); + static List destinations = new ArrayList<>(); + static List vaults = new ArrayList<>(); + static List transforms = new ArrayList<>(); + static Pipeline pipeline; + + @Test + @Order(0) + public void createVault() { + var vault1 = evm.create(Vault.class); + vault1.setName("vault1"); + vault1.setItems(Map.of("foo", "bar", "baz", "qux")); + var vault2 = evm.create(Vault.class); + vault2.setName("vault2"); + vault2.setItems(Map.of("baz", "qux")); + + vaults.add(vaultService.create(vault1)); + vaults.add(vaultService.create(vault2)); + } + + @Test + @Order(10) + public void updateVault() { + var v = vaults.get(0); + v.setItems(Map.of("foo", "bar")); + v = vaultService.update(v); + vaults.set(0, v); + } + + @Test + @Order(20) + public void createSources() { + Assumptions.assumeThat(vaults).hasSize(2); + + var source1 = evm.create(Source.class); + source1.setName("source1"); + source1.setSchema("schemaXY"); + source1.setType("io.debezium.connector.MongoDbConnector"); + source1.setVaults(Set.of(vaultRef(1))); + source1.setConfig(Map.of("mongodb.connection.string", "mongodb://localhost:27017")); + + var source2 = evm.create(Source.class); + source2.setName("source2"); + source2.setSchema("schemaXY"); + source2.setType("io.debezium.connector.MongoDbConnector"); + source2.setVaults(Set.of(vaultRef(0))); + source2.setConfig(Map.of("mongodb.connection.string", "mongodb://localhost:37017")); + + sources.add(sourceService.create(source1)); + sources.add(sourceService.create(source2)); + } + + @Test + @Order(30) + public void createDestinations() { + Assumptions.assumeThat(vaults).hasSize(2); + + var destination1 = evm.create(Destination.class); + destination1.setName("destination1"); + destination1.setSchema("schemaDXY"); + destination1.setType("pubsub"); + destination1.setVaults(Set.of(vaultRef(0),vaultRef(1))); + destination1.setConfig(Map.of("foo", "bar")); + + var destination2 = evm.create(Destination.class); + destination2.setName("destination2"); + destination2.setSchema("schemaDYZ"); + destination2.setType("redis"); + destination2.setVaults(Set.of(vaultRef(0))); + destination2.setConfig(Map.of("bar", "baz")); + + destinations.add(destinationService.create(destination1)); + destinations.add(destinationService.create(destination2)); + } + + @Test + @Order(40) + public void createTransform() { + Assumptions.assumeThat(vaults).hasSize(2); + + var transform1 = evm.create(Transform.class); + transform1.setName("transform1"); + transform1.setSchema("schemaASD"); + transform1.setType("io.example.SomeTransform"); + transform1.setVaults(Set.of(vaultRef(0),vaultRef(1))); + transform1.setConfig(Map.of("baz", "qux")); + + transforms.add(transformService.create(transform1)); + } + + @Test + @Order(50) + public void createPipeline() { + Assumptions.assumeThat(sources).isNotEmpty(); + Assumptions.assumeThat(destinations).isNotEmpty(); + Assumptions.assumeThat(transforms).isNotEmpty(); + + var sourceRef = sourceRef(0); + var destinationRef = destinationRef(0); + + var newPipeline = evm.create(Pipeline.class); + newPipeline.setName("pipeline1"); + newPipeline.setSource(sourceRef); + newPipeline.setDestination(destinationRef); + newPipeline.setLogLevel("info"); + + pipeline = pipelineService.create(newPipeline); + } + + @Test + @Order(51) + public void updatePipeline() { + Assumptions.assumeThat(sources).hasSize(2); + Assumptions.assumeThat(destinations).hasSize(2); + Assumptions.assumeThat(pipeline).isNotNull(); + + + var sourceRef = sourceRef(1); + var destinationRef = destinationRef(1); + var uPipeline = pipelineService.findById(pipeline.getId()).orElse(null); + uPipeline.setSource(sourceRef); + uPipeline.setDestination(destinationRef); + + pipeline = pipelineService.update(uPipeline); + } + + @Test + @Order(52) + public void listPipelines() { + var list = pipelineService.list(); + + Assertions.assertThat(list).hasSize(1); + } + + @Test + @Order(61) + public void deletePipeline() { + pipelineService.delete(pipeline.getId()); + + var found = pipelineService.findById(pipeline.getId()); + Assertions.assertThat(found).isEmpty(); + Assertions.assertThat(destinationService.list()).hasSize(2); + Assertions.assertThat(sourceService.list()).hasSize(2); + } + + @Test + @Order(100) + public void foo() { + System.out.println(); + } + + public VaultReference vaultRef(int idx) { + return vaultService.viewAs(vaults.get(idx), VaultReference.class); + } + + public SourceReference sourceRef(int idx) { + return sourceService.viewAs(sources.get(idx), SourceReference.class); + } + + public DestinationReference destinationRef(int idx) { + return destinationService.viewAs(destinations.get(idx), DestinationReference.class); + } +}