From 98b6e4c38c178c82da8e1dda038865d07b6d14cc Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Wed, 24 Apr 2024 15:35:10 +0200 Subject: [PATCH 01/44] Project bootstrap --- .dockerignore | 5 + .gitignore | 43 ++++ .mvn/wrapper/.gitignore | 1 + .mvn/wrapper/MavenWrapperDownloader.java | 98 ++++++++ .mvn/wrapper/maven-wrapper.properties | 18 ++ README.md | 52 ++++ mvnw | 308 +++++++++++++++++++++++ mvnw.cmd | 205 +++++++++++++++ pom.xml | 120 +++++++++ src/main/docker/Dockerfile.jvm | 97 +++++++ src/main/docker/Dockerfile.legacy-jar | 93 +++++++ src/main/docker/Dockerfile.native | 27 ++ src/main/docker/Dockerfile.native-micro | 30 +++ src/main/resources/application.yml | 1 + 14 files changed, 1098 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .mvn/wrapper/.gitignore create mode 100644 .mvn/wrapper/MavenWrapperDownloader.java create mode 100644 .mvn/wrapper/maven-wrapper.properties create mode 100644 README.md create mode 100755 mvnw create mode 100644 mvnw.cmd create mode 100644 pom.xml create mode 100644 src/main/docker/Dockerfile.jvm create mode 100644 src/main/docker/Dockerfile.legacy-jar create mode 100644 src/main/docker/Dockerfile.native create mode 100644 src/main/docker/Dockerfile.native-micro create mode 100644 src/main/resources/application.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..94810d00 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +* +!target/*-runner +!target/*-runner.jar +!target/lib/* +!target/quarkus-app/* \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..8c7863e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,43 @@ +#Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +release.properties +.flattened-pom.xml + +# Eclipse +.project +.classpath +.settings/ +bin/ + +# IntelliJ +.idea +*.ipr +*.iml +*.iws + +# NetBeans +nb-configuration.xml + +# Visual Studio Code +.vscode +.factorypath + +# OSX +.DS_Store + +# Vim +*.swp +*.swo + +# patch +*.orig +*.rej + +# Local environment +.env + +# Plugin directory +/.quarkus/cli/plugins/ diff --git a/.mvn/wrapper/.gitignore b/.mvn/wrapper/.gitignore new file mode 100644 index 00000000..e72f5e8b --- /dev/null +++ b/.mvn/wrapper/.gitignore @@ -0,0 +1 @@ +maven-wrapper.jar diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 00000000..84d1e60d --- /dev/null +++ b/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +public final class MavenWrapperDownloader +{ + private static final String WRAPPER_VERSION = "3.2.0"; + + private static final boolean VERBOSE = Boolean.parseBoolean( System.getenv( "MVNW_VERBOSE" ) ); + + public static void main( String[] args ) + { + log( "Apache Maven Wrapper Downloader " + WRAPPER_VERSION ); + + if ( args.length != 2 ) + { + System.err.println( " - ERROR wrapperUrl or wrapperJarPath parameter missing" ); + System.exit( 1 ); + } + + try + { + log( " - Downloader started" ); + final URL wrapperUrl = new URL( args[0] ); + final String jarPath = args[1].replace( "..", "" ); // Sanitize path + final Path wrapperJarPath = Paths.get( jarPath ).toAbsolutePath().normalize(); + downloadFileFromURL( wrapperUrl, wrapperJarPath ); + log( "Done" ); + } + catch ( IOException e ) + { + System.err.println( "- Error downloading: " + e.getMessage() ); + if ( VERBOSE ) + { + e.printStackTrace(); + } + System.exit( 1 ); + } + } + + private static void downloadFileFromURL( URL wrapperUrl, Path wrapperJarPath ) + throws IOException + { + log( " - Downloading to: " + wrapperJarPath ); + if ( System.getenv( "MVNW_USERNAME" ) != null && System.getenv( "MVNW_PASSWORD" ) != null ) + { + final String username = System.getenv( "MVNW_USERNAME" ); + final char[] password = System.getenv( "MVNW_PASSWORD" ).toCharArray(); + Authenticator.setDefault( new Authenticator() + { + @Override + protected PasswordAuthentication getPasswordAuthentication() + { + return new PasswordAuthentication( username, password ); + } + } ); + } + try ( InputStream inStream = wrapperUrl.openStream() ) + { + Files.copy( inStream, wrapperJarPath, StandardCopyOption.REPLACE_EXISTING ); + } + log( " - Downloader complete" ); + } + + private static void log( String msg ) + { + if ( VERBOSE ) + { + System.out.println( msg ); + } + } + +} diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..346d645f --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/README.md b/README.md new file mode 100644 index 00000000..64fa57b8 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +# debezium-platform-conductor + +This project uses Quarkus, the Supersonic Subatomic Java Framework. + +If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ . + +## Running the application in dev mode + +You can run your application in dev mode that enables live coding using: +```shell script +./mvnw compile quarkus:dev +``` + +> **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/. + +## Packaging and running the application + +The application can be packaged using: +```shell script +./mvnw package +``` +It produces the `quarkus-run.jar` file in the `target/quarkus-app/` directory. +Be aware that it’s not an _über-jar_ as the dependencies are copied into the `target/quarkus-app/lib/` directory. + +The application is now runnable using `java -jar target/quarkus-app/quarkus-run.jar`. + +If you want to build an _über-jar_, execute the following command: +```shell script +./mvnw package -Dquarkus.package.type=uber-jar +``` + +The application, packaged as an _über-jar_, is now runnable using `java -jar target/*-runner.jar`. + +## Creating a native executable + +You can create a native executable using: +```shell script +./mvnw package -Dnative +``` + +Or, if you don't have GraalVM installed, you can run the native executable build in a container using: +```shell script +./mvnw package -Dnative -Dquarkus.native.container-build=true +``` + +You can then execute your native executable with: `./target/debezium-platform-conductor-0.1.0-SNAPSHOT-runner` + +If you want to learn more about building native executables, please consult https://quarkus.io/guides/maven-tooling. + +## Related Guides + +- YAML Configuration ([guide](https://quarkus.io/guides/config-yaml)): Use YAML to configure your Quarkus application diff --git a/mvnw b/mvnw new file mode 100755 index 00000000..8d937f4c --- /dev/null +++ b/mvnw @@ -0,0 +1,308 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.2.0 +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "$(uname)" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME + else + JAVA_HOME="/Library/Java/Home"; export JAVA_HOME + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && + JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then + if $darwin ; then + javaHome="$(dirname "\"$javaExecutable\"")" + javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" + else + javaExecutable="$(readlink -f "\"$javaExecutable\"")" + fi + javaHome="$(dirname "\"$javaExecutable\"")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$(cd "$wdir/.." || exit 1; pwd) + fi + # end of workaround + done + printf '%s' "$(cd "$basedir" || exit 1; pwd)" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + # Remove \r in case we run on Windows within Git Bash + # and check out the repository with auto CRLF management + # enabled. Otherwise, we may read lines that are delimited with + # \r\n and produce $'-Xarg\r' rather than -Xarg due to word + # splitting rules. + tr -s '\r\n' ' ' < "$1" + fi +} + +log() { + if [ "$MVNW_VERBOSE" = true ]; then + printf '%s\n' "$1" + fi +} + +BASE_DIR=$(find_maven_basedir "$(dirname "$0")") +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR +log "$MAVEN_PROJECTBASEDIR" + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" +if [ -r "$wrapperJarPath" ]; then + log "Found $wrapperJarPath" +else + log "Couldn't find $wrapperJarPath, downloading it ..." + + if [ -n "$MVNW_REPOURL" ]; then + wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + else + wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + fi + while IFS="=" read -r key value; do + # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) + safeValue=$(echo "$value" | tr -d '\r') + case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; + esac + done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" + log "Downloading from: $wrapperUrl" + + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget > /dev/null; then + log "Found wget ... using wget" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + log "Found curl ... using curl" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + else + curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + fi + else + log "Falling back to using Java to download" + javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" + javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaSource=$(cygpath --path --windows "$javaSource") + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaSource" ]; then + if [ ! -e "$javaClass" ]; then + log " - Compiling MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/javac" "$javaSource") + fi + if [ -e "$javaClass" ]; then + log " - Running MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +# If specified, validate the SHA-256 sum of the Maven wrapper jar file +wrapperSha256Sum="" +while IFS="=" read -r key value; do + case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; + esac +done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" +if [ -n "$wrapperSha256Sum" ]; then + wrapperSha256Result=false + if command -v sha256sum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." + echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + exit 1 + fi + if [ $wrapperSha256Result = false ]; then + echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 + echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 + echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + exit 1 + fi +fi + +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# shellcheck disable=SC2086 # safe args +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 00000000..c4586b56 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,205 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.2.0 +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* +if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %WRAPPER_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file +SET WRAPPER_SHA_256_SUM="" +FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B +) +IF NOT %WRAPPER_SHA_256_SUM%=="" ( + powershell -Command "&{"^ + "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ + "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ + " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ + " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ + " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ + " exit 1;"^ + "}"^ + "}" + if ERRORLEVEL 1 goto error +) + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% ^ + %JVM_CONFIG_MAVEN_PROPS% ^ + %MAVEN_OPTS% ^ + %MAVEN_DEBUG_OPTS% ^ + -classpath %WRAPPER_JAR% ^ + "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ + %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" +if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%"=="on" pause + +if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% + +cmd /C exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..62f30644 --- /dev/null +++ b/pom.xml @@ -0,0 +1,120 @@ + + + 4.0.0 + io.debezium + debezium-platform-conductor + 0.1.0-SNAPSHOT + + 3.12.1 + 21 + UTF-8 + UTF-8 + quarkus-bom + io.quarkus.platform + 3.8.4 + true + 3.2.5 + + + + + ${quarkus.platform.group-id} + ${quarkus.platform.artifact-id} + ${quarkus.platform.version} + pom + import + + + + + + io.quarkus + quarkus-resteasy-reactive-jackson + + + io.quarkus + quarkus-config-yaml + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-junit5 + test + + + + + + ${quarkus.platform.group-id} + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + + build + generate-code + generate-code-tests + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + + -parameters + + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + maven-failsafe-plugin + ${surefire-plugin.version} + + + + integration-test + verify + + + + + + ${project.build.directory}/${project.build.finalName}-runner + org.jboss.logmanager.LogManager + ${maven.home} + + + + + + + + native + + + native + + + + false + native + + + + diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm new file mode 100644 index 00000000..5444af0a --- /dev/null +++ b/src/main/docker/Dockerfile.jvm @@ -0,0 +1,97 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./mvnw package +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.jvm -t quarkus/debezium-platform-conductor-jvm . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-jvm +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-jvm +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM registry.access.redhat.com/ubi8/openjdk-21:1.18 + +ENV LANGUAGE='en_US:en' + + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --chown=185 target/quarkus-app/lib/ /deployments/lib/ +COPY --chown=185 target/quarkus-app/*.jar /deployments/ +COPY --chown=185 target/quarkus-app/app/ /deployments/app/ +COPY --chown=185 target/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8080 +USER 185 +ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] + diff --git a/src/main/docker/Dockerfile.legacy-jar b/src/main/docker/Dockerfile.legacy-jar new file mode 100644 index 00000000..c952d706 --- /dev/null +++ b/src/main/docker/Dockerfile.legacy-jar @@ -0,0 +1,93 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode +# +# Before building the container image run: +# +# ./mvnw package -Dquarkus.package.type=legacy-jar +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.legacy-jar -t quarkus/debezium-platform-conductor-legacy-jar . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-legacy-jar +# +# If you want to include the debug port into your docker image +# you will have to expose the debug port (default 5005 being the default) like this : EXPOSE 8080 5005. +# Additionally you will have to set -e JAVA_DEBUG=true and -e JAVA_DEBUG_PORT=*:5005 +# when running the container +# +# Then run the container using : +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor-legacy-jar +# +# This image uses the `run-java.sh` script to run the application. +# This scripts computes the command line to execute your Java application, and +# includes memory/GC tuning. +# You can configure the behavior using the following environment properties: +# - JAVA_OPTS: JVM options passed to the `java` command (example: "-verbose:class") +# - JAVA_OPTS_APPEND: User specified Java options to be appended to generated options +# in JAVA_OPTS (example: "-Dsome.property=foo") +# - JAVA_MAX_MEM_RATIO: Is used when no `-Xmx` option is given in JAVA_OPTS. This is +# used to calculate a default maximal heap memory based on a containers restriction. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xmx` is set to a ratio +# of the container available memory as set here. The default is `50` which means 50% +# of the available memory is used as an upper boundary. You can skip this mechanism by +# setting this value to `0` in which case no `-Xmx` option is added. +# - JAVA_INITIAL_MEM_RATIO: Is used when no `-Xms` option is given in JAVA_OPTS. This +# is used to calculate a default initial heap memory based on the maximum heap memory. +# If used in a container without any memory constraints for the container then this +# option has no effect. If there is a memory constraint then `-Xms` is set to a ratio +# of the `-Xmx` memory as set here. The default is `25` which means 25% of the `-Xmx` +# is used as the initial heap size. You can skip this mechanism by setting this value +# to `0` in which case no `-Xms` option is added (example: "25") +# - JAVA_MAX_INITIAL_MEM: Is used when no `-Xms` option is given in JAVA_OPTS. +# This is used to calculate the maximum value of the initial heap memory. If used in +# a container without any memory constraints for the container then this option has +# no effect. If there is a memory constraint then `-Xms` is limited to the value set +# here. The default is 4096MB which means the calculated value of `-Xms` never will +# be greater than 4096MB. The value of this variable is expressed in MB (example: "4096") +# - JAVA_DIAGNOSTICS: Set this to get some diagnostics information to standard output +# when things are happening. This option, if set to true, will set +# `-XX:+UnlockDiagnosticVMOptions`. Disabled by default (example: "true"). +# - JAVA_DEBUG: If set remote debugging will be switched on. Disabled by default (example: +# true"). +# - JAVA_DEBUG_PORT: Port used for remote debugging. Defaults to 5005 (example: "8787"). +# - CONTAINER_CORE_LIMIT: A calculated core limit as described in +# https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt. (example: "2") +# - CONTAINER_MAX_MEMORY: Memory limit given to the container (example: "1024"). +# - GC_MIN_HEAP_FREE_RATIO: Minimum percentage of heap free after GC to avoid expansion. +# (example: "20") +# - GC_MAX_HEAP_FREE_RATIO: Maximum percentage of heap free after GC to avoid shrinking. +# (example: "40") +# - GC_TIME_RATIO: Specifies the ratio of the time spent outside the garbage collection. +# (example: "4") +# - GC_ADAPTIVE_SIZE_POLICY_WEIGHT: The weighting given to the current GC time versus +# previous GC times. (example: "90") +# - GC_METASPACE_SIZE: The initial metaspace size. (example: "20") +# - GC_MAX_METASPACE_SIZE: The maximum metaspace size. (example: "100") +# - GC_CONTAINER_OPTIONS: Specify Java GC to use. The value of this variable should +# contain the necessary JRE command-line options to specify the required GC, which +# will override the default of `-XX:+UseParallelGC` (example: -XX:+UseG1GC). +# - HTTPS_PROXY: The location of the https proxy. (example: "myuser@127.0.0.1:8080") +# - HTTP_PROXY: The location of the http proxy. (example: "myuser@127.0.0.1:8080") +# - NO_PROXY: A comma separated lists of hosts, IP addresses or domains that can be +# accessed directly. (example: "foo.example.com,bar.example.com") +# +### +FROM registry.access.redhat.com/ubi8/openjdk-21:1.18 + +ENV LANGUAGE='en_US:en' + + +COPY target/lib/* /deployments/lib/ +COPY target/*-runner.jar /deployments/quarkus-run.jar + +EXPOSE 8080 +USER 185 +ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" + +ENTRYPOINT [ "/opt/jboss/container/java/run/run-java.sh" ] diff --git a/src/main/docker/Dockerfile.native b/src/main/docker/Dockerfile.native new file mode 100644 index 00000000..4785884f --- /dev/null +++ b/src/main/docker/Dockerfile.native @@ -0,0 +1,27 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# +# Before building the container image run: +# +# ./mvnw package -Dnative +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native -t quarkus/debezium-platform-conductor . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor +# +### +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root target/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"] diff --git a/src/main/docker/Dockerfile.native-micro b/src/main/docker/Dockerfile.native-micro new file mode 100644 index 00000000..aa0c26e3 --- /dev/null +++ b/src/main/docker/Dockerfile.native-micro @@ -0,0 +1,30 @@ +#### +# This Dockerfile is used in order to build a container that runs the Quarkus application in native (no JVM) mode. +# It uses a micro base image, tuned for Quarkus native executables. +# It reduces the size of the resulting container image. +# Check https://quarkus.io/guides/quarkus-runtime-base-image for further information about this image. +# +# Before building the container image run: +# +# ./mvnw package -Dnative +# +# Then, build the image with: +# +# docker build -f src/main/docker/Dockerfile.native-micro -t quarkus/debezium-platform-conductor . +# +# Then run the container using: +# +# docker run -i --rm -p 8080:8080 quarkus/debezium-platform-conductor +# +### +FROM quay.io/quarkus/quarkus-micro-image:2.0 +WORKDIR /work/ +RUN chown 1001 /work \ + && chmod "g+rwX" /work \ + && chown 1001:root /work +COPY --chown=1001:root target/*-runner /work/application + +EXPOSE 8080 +USER 1001 + +ENTRYPOINT ["./application", "-Dquarkus.http.host=0.0.0.0"] diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1 @@ +{} From 170353c1e8b4897ed59bfd1c09fc7710d0ac42f3 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 7 May 2024 13:49:05 +0200 Subject: [PATCH 02/44] Initial CRUD bootstrap --- pom.xml | 74 ++++++- .../data/model/DestinationEntity.java | 37 ++++ .../platform/data/model/PipelineEntity.java | 36 ++++ .../platform/data/model/SourceEntity.java | 37 ++++ .../platform/data/model/TransformEntity.java | 34 +++ .../platform/data/model/VaultEntity.java | 28 +++ .../platform/domain/AbstractService.java | 74 +++++++ .../platform/domain/DestinationService.java | 29 +++ .../platform/domain/PipelineService.java | 17 ++ .../platform/domain/SourceService.java | 34 +++ .../platform/domain/TransformService.java | 31 +++ .../platform/domain/VaultService.java | 30 +++ .../platform/domain/views/Destination.java | 12 ++ .../platform/domain/views/Pipeline.java | 31 +++ .../domain/views/PipelineComponent.java | 25 +++ .../platform/domain/views/Source.java | 13 ++ .../platform/domain/views/Transform.java | 12 ++ .../debezium/platform/domain/views/Vault.java | 21 ++ .../platform/domain/views/base/IdView.java | 8 + .../platform/domain/views/base/NamedView.java | 8 + .../views/refs/DestinationReference.java | 9 + .../domain/views/refs/SourceReference.java | 9 + .../domain/views/refs/TransformReference.java | 9 + .../domain/views/refs/VaultReference.java | 9 + src/main/resources/application.yml | 7 +- .../io/debezium/platform/ServiceTest.java | 202 ++++++++++++++++++ 26 files changed, 833 insertions(+), 3 deletions(-) create mode 100644 src/main/java/io/debezium/platform/data/model/DestinationEntity.java create mode 100644 src/main/java/io/debezium/platform/data/model/PipelineEntity.java create mode 100644 src/main/java/io/debezium/platform/data/model/SourceEntity.java create mode 100644 src/main/java/io/debezium/platform/data/model/TransformEntity.java create mode 100644 src/main/java/io/debezium/platform/data/model/VaultEntity.java create mode 100644 src/main/java/io/debezium/platform/domain/AbstractService.java create mode 100644 src/main/java/io/debezium/platform/domain/DestinationService.java create mode 100644 src/main/java/io/debezium/platform/domain/PipelineService.java create mode 100644 src/main/java/io/debezium/platform/domain/SourceService.java create mode 100644 src/main/java/io/debezium/platform/domain/TransformService.java create mode 100644 src/main/java/io/debezium/platform/domain/VaultService.java create mode 100644 src/main/java/io/debezium/platform/domain/views/Destination.java create mode 100644 src/main/java/io/debezium/platform/domain/views/Pipeline.java create mode 100644 src/main/java/io/debezium/platform/domain/views/PipelineComponent.java create mode 100644 src/main/java/io/debezium/platform/domain/views/Source.java create mode 100644 src/main/java/io/debezium/platform/domain/views/Transform.java create mode 100644 src/main/java/io/debezium/platform/domain/views/Vault.java create mode 100644 src/main/java/io/debezium/platform/domain/views/base/IdView.java create mode 100644 src/main/java/io/debezium/platform/domain/views/base/NamedView.java create mode 100644 src/main/java/io/debezium/platform/domain/views/refs/DestinationReference.java create mode 100644 src/main/java/io/debezium/platform/domain/views/refs/SourceReference.java create mode 100644 src/main/java/io/debezium/platform/domain/views/refs/TransformReference.java create mode 100644 src/main/java/io/debezium/platform/domain/views/refs/VaultReference.java create mode 100644 src/test/java/io/debezium/platform/ServiceTest.java diff --git a/pom.xml b/pom.xml index 62f30644..a8cf2294 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,9 @@ 3.8.4 true 3.2.5 + 3.24.2 + 1.18.32 + 1.5.5.Final @@ -25,26 +28,93 @@ pom import + + ${quarkus.platform.group-id} + quarkus-blaze-persistence-bom + ${quarkus.platform.version} + pom + import + + + io.quarkus + quarkus-config-yaml + + + io.quarkus + quarkus-arc + io.quarkus quarkus-resteasy-reactive-jackson io.quarkus - quarkus-config-yaml + quarkus-hibernate-orm io.quarkus - quarkus-arc + quarkus-hibernate-validator + + + io.quarkus + quarkus-jdbc-postgresql + + + io.quarkus + quarkus-smallrye-openapi + + + + com.blazebit + blaze-persistence-integration-quarkus-3 + + + com.blazebit + blaze-persistence-integration-hibernate-6.2 + runtime + + + com.blazebit + blaze-persistence-integration-jackson-jakarta + 1.6.11 + + + com.blazebit + blaze-persistence-integration-jaxrs-jackson-jakarta + 1.6.11 + + + + + + + + + + org.projectlombok + lombok + ${version.lombok} + provided io.quarkus quarkus-junit5 test + + org.assertj + assertj-core + ${version.assertj} + test + + + org.mapstruct + mapstruct + ${version.mapstruct} + diff --git a/src/main/java/io/debezium/platform/data/model/DestinationEntity.java b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java new file mode 100644 index 00000000..9dc55c77 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java @@ -0,0 +1,37 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +@Entity(name = "Destination") +@Getter +@Setter +public class DestinationEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + private String name; + @NotEmpty + private String type; + @NotEmpty + private String schema; + @ManyToMany + @JoinTable(inverseJoinColumns = @JoinColumn(name = "vault_id")) + private List vaults = new LinkedList<>(); + @JdbcTypeCode(SqlTypes.JSON) + private Map config; +} diff --git a/src/main/java/io/debezium/platform/data/model/PipelineEntity.java b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java new file mode 100644 index 00000000..1c6595f5 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java @@ -0,0 +1,36 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; +import jakarta.persistence.ManyToOne; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; + +import java.util.LinkedList; +import java.util.List; + + +@Entity(name = "Pipeline") +@Getter +@Setter +public class PipelineEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + private String name; + @ManyToOne + private SourceEntity source; + @ManyToOne + private DestinationEntity destination; + @ManyToMany + @JoinTable(inverseJoinColumns = @JoinColumn(name = "transform_id")) + private List transforms = new LinkedList<>(); + @NotEmpty + private String logLevel = "info"; +} diff --git a/src/main/java/io/debezium/platform/data/model/SourceEntity.java b/src/main/java/io/debezium/platform/data/model/SourceEntity.java new file mode 100644 index 00000000..d11b0d64 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/SourceEntity.java @@ -0,0 +1,37 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; +import jakarta.persistence.ManyToMany; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +@Entity(name = "Source") +@Getter +@Setter +public class SourceEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + public String name; + @NotEmpty + public String type; + @NotEmpty + public String schema; + @ManyToMany + @JoinTable(inverseJoinColumns = @JoinColumn(name = "vault_id")) + public List vaults = new LinkedList<>(); + @JdbcTypeCode(SqlTypes.JSON) + public Map config; +} diff --git a/src/main/java/io/debezium/platform/data/model/TransformEntity.java b/src/main/java/io/debezium/platform/data/model/TransformEntity.java new file mode 100644 index 00000000..82557349 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/TransformEntity.java @@ -0,0 +1,34 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.persistence.ManyToMany; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +@Entity(name = "Transform") +@Getter +@Setter +public class TransformEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + private String name; + @NotEmpty + private String type; + @NotEmpty + private String schema; + @ManyToMany + private List vaults = new LinkedList<>(); + @JdbcTypeCode(SqlTypes.JSON) + private Map config; +} diff --git a/src/main/java/io/debezium/platform/data/model/VaultEntity.java b/src/main/java/io/debezium/platform/data/model/VaultEntity.java new file mode 100644 index 00000000..ecac75d0 --- /dev/null +++ b/src/main/java/io/debezium/platform/data/model/VaultEntity.java @@ -0,0 +1,28 @@ +package io.debezium.platform.data.model; + +import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.Id; +import jakarta.validation.constraints.NotEmpty; +import lombok.Getter; +import lombok.Setter; + +import java.util.LinkedList; +import java.util.List; + +@Entity(name = "Vault") +@Getter +@Setter +public class VaultEntity { + @Id + @GeneratedValue + private Long id; + @NotEmpty + private String name; + private boolean plaintext = false; + @ElementCollection + @Column(name = "key") + private List keys = new LinkedList<>(); +} diff --git a/src/main/java/io/debezium/platform/domain/AbstractService.java b/src/main/java/io/debezium/platform/domain/AbstractService.java new file mode 100644 index 00000000..66eb3c6b --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/AbstractService.java @@ -0,0 +1,74 @@ +package io.debezium.platform.domain; + +import com.blazebit.persistence.CriteriaBuilder; +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import com.blazebit.persistence.view.EntityViewSetting; +import io.debezium.platform.domain.views.base.IdView; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import jakarta.validation.Valid; + +import java.util.List; +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.REQUIRED; +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +@Transactional(REQUIRED) +public class AbstractService { + + EntityManager em; + CriteriaBuilderFactory cbf; + EntityViewManager evm; + Class viewType; + Class entityType; + + public AbstractService( + Class entityType, + Class viewType, + EntityManager em, + CriteriaBuilderFactory cbf, + EntityViewManager evm) { + this.em = em; + this.cbf = cbf; + this.evm = evm; + this.entityType = entityType; + this.viewType = viewType; + } + + public AbstractService() { + // required by CDI + } + + protected CriteriaBuilder cb() { + return cbf.create(em, entityType); + } + + @Transactional(SUPPORTS) + public List list() { + return evm.applySetting(EntityViewSetting.create(viewType), cb()) + .getResultList(); + } + + @Transactional(SUPPORTS) + public Optional findById(Long id) { + var result = evm.find(em, viewType, id); + return Optional.ofNullable(result); + } + + public T create(@Valid T view) { + evm.save(em, view); + return view; + } + + public T update(@Valid T view) { + evm.save(em, view); + return evm.find(em, viewType, view.getId()); + } + + public void delete(long id) { + evm.remove(em, id); + } + +} diff --git a/src/main/java/io/debezium/platform/domain/DestinationService.java b/src/main/java/io/debezium/platform/domain/DestinationService.java new file mode 100644 index 00000000..2d900bee --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/DestinationService.java @@ -0,0 +1,29 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.data.model.DestinationEntity; +import io.debezium.platform.domain.views.Destination; +import io.debezium.platform.domain.views.refs.DestinationReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +@ApplicationScoped +public class DestinationService extends AbstractService { + + public DestinationService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(DestinationEntity.class, Destination.class, em, cbf, evm); + } + + @Transactional(SUPPORTS) + public Optional findReferenceById(Long id) { + var result = evm.find(em, DestinationReference.class, id); + return Optional.ofNullable(result); + } +} diff --git a/src/main/java/io/debezium/platform/domain/PipelineService.java b/src/main/java/io/debezium/platform/domain/PipelineService.java new file mode 100644 index 00000000..beddf2ab --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/PipelineService.java @@ -0,0 +1,17 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.Pipeline; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; + +@ApplicationScoped +public class PipelineService extends AbstractService { + + public PipelineService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(PipelineEntity.class, Pipeline.class, em, cbf, evm); + } +} diff --git a/src/main/java/io/debezium/platform/domain/SourceService.java b/src/main/java/io/debezium/platform/domain/SourceService.java new file mode 100644 index 00000000..c13d35ee --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/SourceService.java @@ -0,0 +1,34 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import com.blazebit.persistence.view.EntityViewSetting; +import io.debezium.platform.data.model.SourceEntity; +import io.debezium.platform.domain.views.Source; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.domain.views.refs.SourceReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import jakarta.validation.Valid; + +import java.util.List; +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.REQUIRED; +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +@ApplicationScoped +public class SourceService extends AbstractService { + + public SourceService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(SourceEntity.class, Source.class, em, cbf, evm); + } + + @Transactional(SUPPORTS) + public Optional findReferenceById(Long id) { + var result = evm.find(em, SourceReference.class, id); + return Optional.ofNullable(result); + } +} diff --git a/src/main/java/io/debezium/platform/domain/TransformService.java b/src/main/java/io/debezium/platform/domain/TransformService.java new file mode 100644 index 00000000..77d3b03c --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/TransformService.java @@ -0,0 +1,31 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.data.model.TransformEntity; +import io.debezium.platform.domain.views.Transform; +import io.debezium.platform.domain.views.refs.SourceReference; +import io.debezium.platform.domain.views.refs.TransformReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.REQUIRED; +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + +@ApplicationScoped +public class TransformService extends AbstractService { + + public TransformService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(TransformEntity.class, Transform.class, em, cbf, evm); + } + + @Transactional(SUPPORTS) + public Optional findReferenceById(Long id) { + var result = evm.find(em, TransformReference.class, id); + return Optional.ofNullable(result); + } +} diff --git a/src/main/java/io/debezium/platform/domain/VaultService.java b/src/main/java/io/debezium/platform/domain/VaultService.java new file mode 100644 index 00000000..80163b62 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/VaultService.java @@ -0,0 +1,30 @@ +package io.debezium.platform.domain; + + +import com.blazebit.persistence.CriteriaBuilderFactory; +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.data.model.VaultEntity; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.domain.views.refs.VaultReference; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.Optional; + +import static jakarta.transaction.Transactional.TxType.SUPPORTS; + + +@ApplicationScoped +public class VaultService extends AbstractService { + + public VaultService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + super(VaultEntity.class, Vault.class, em, cbf, evm); + } + + @Transactional(SUPPORTS) + public Optional findReferenceById(Long id) { + var result = evm.find(em, VaultReference.class, id); + return Optional.ofNullable(result); + } +} diff --git a/src/main/java/io/debezium/platform/domain/views/Destination.java b/src/main/java/io/debezium/platform/domain/views/Destination.java new file mode 100644 index 00000000..0a2cf683 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Destination.java @@ -0,0 +1,12 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.DestinationEntity; + +@EntityView(DestinationEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Destination extends PipelineComponent { +} diff --git a/src/main/java/io/debezium/platform/domain/views/Pipeline.java b/src/main/java/io/debezium/platform/domain/views/Pipeline.java new file mode 100644 index 00000000..9af5042d --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Pipeline.java @@ -0,0 +1,31 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.base.NamedView; +import io.debezium.platform.domain.views.refs.DestinationReference; +import io.debezium.platform.domain.views.refs.SourceReference; +import io.debezium.platform.domain.views.refs.TransformReference; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +import java.util.List; + +@EntityView(PipelineEntity.class) +@CreatableEntityView +public interface Pipeline extends NamedView { + @NotNull + SourceReference getSource(); + @NotNull + DestinationReference getDestination(); + List getTransforms(); + @NotEmpty + String getLogLevel(); + + void setName(String name); + void setSource(SourceReference source); + void setDestination(DestinationReference destination); + void setLogLevel(String logLevel); + void setTransforms(List transforms); +} diff --git a/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java new file mode 100644 index 00000000..d58eda28 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java @@ -0,0 +1,25 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.MappingSingular; +import io.debezium.platform.domain.views.base.NamedView; +import io.debezium.platform.domain.views.refs.VaultReference; +import jakarta.validation.constraints.NotEmpty; + +import java.util.List; +import java.util.Map; + +public interface PipelineComponent extends NamedView { + @NotEmpty + String getType(); + @NotEmpty + String getSchema(); + List getVaults(); + @MappingSingular + Map getConfig(); + + void setType(String type); + void setName(String name); + void setSchema(String schema); + void setVaults(List vaults); + void setConfig(Map config); +} diff --git a/src/main/java/io/debezium/platform/domain/views/Source.java b/src/main/java/io/debezium/platform/domain/views/Source.java new file mode 100644 index 00000000..819adb04 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Source.java @@ -0,0 +1,13 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.SourceEntity; + + +@EntityView(SourceEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Source extends PipelineComponent { +} diff --git a/src/main/java/io/debezium/platform/domain/views/Transform.java b/src/main/java/io/debezium/platform/domain/views/Transform.java new file mode 100644 index 00000000..4dc20b01 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Transform.java @@ -0,0 +1,12 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.TransformEntity; + +@EntityView(TransformEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Transform extends PipelineComponent { +} diff --git a/src/main/java/io/debezium/platform/domain/views/Vault.java b/src/main/java/io/debezium/platform/domain/views/Vault.java new file mode 100644 index 00000000..9a6dc45e --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/Vault.java @@ -0,0 +1,21 @@ +package io.debezium.platform.domain.views; + +import com.blazebit.persistence.view.CreatableEntityView; +import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; +import io.debezium.platform.data.model.VaultEntity; +import io.debezium.platform.domain.views.refs.VaultReference; + +import java.util.List; + +@EntityView(VaultEntity.class) +@CreatableEntityView +@UpdatableEntityView +public interface Vault extends VaultReference { + boolean isPlaintext(); + List getKeys(); + + void setName(String name); + void setPlaintext(boolean plaintext); + void setKeys(List keys); +} diff --git a/src/main/java/io/debezium/platform/domain/views/base/IdView.java b/src/main/java/io/debezium/platform/domain/views/base/IdView.java new file mode 100644 index 00000000..96835bfd --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/base/IdView.java @@ -0,0 +1,8 @@ +package io.debezium.platform.domain.views.base; + +import com.blazebit.persistence.view.IdMapping; + +public interface IdView { + @IdMapping + Long getId(); +} diff --git a/src/main/java/io/debezium/platform/domain/views/base/NamedView.java b/src/main/java/io/debezium/platform/domain/views/base/NamedView.java new file mode 100644 index 00000000..1c69d55b --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/base/NamedView.java @@ -0,0 +1,8 @@ +package io.debezium.platform.domain.views.base; + +import jakarta.validation.constraints.NotEmpty; + +public interface NamedView extends IdView { + @NotEmpty + String getName(); +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/DestinationReference.java b/src/main/java/io/debezium/platform/domain/views/refs/DestinationReference.java new file mode 100644 index 00000000..e0c8a0ff --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/DestinationReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.DestinationEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(DestinationEntity.class) +public interface DestinationReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/SourceReference.java b/src/main/java/io/debezium/platform/domain/views/refs/SourceReference.java new file mode 100644 index 00000000..2217f737 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/SourceReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.SourceEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(SourceEntity.class) +public interface SourceReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/TransformReference.java b/src/main/java/io/debezium/platform/domain/views/refs/TransformReference.java new file mode 100644 index 00000000..83ee9fbe --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/TransformReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.TransformEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(TransformEntity.class) +public interface TransformReference extends NamedView { +} diff --git a/src/main/java/io/debezium/platform/domain/views/refs/VaultReference.java b/src/main/java/io/debezium/platform/domain/views/refs/VaultReference.java new file mode 100644 index 00000000..230c141d --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/VaultReference.java @@ -0,0 +1,9 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.VaultEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(VaultEntity.class) +public interface VaultReference extends NamedView { +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 0967ef42..f1f01031 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1 +1,6 @@ -{} +quarkus: + datasource: + db-kid: postgresql + devservices: + enabled: true + port: 5432 diff --git a/src/test/java/io/debezium/platform/ServiceTest.java b/src/test/java/io/debezium/platform/ServiceTest.java new file mode 100644 index 00000000..3870bfb5 --- /dev/null +++ b/src/test/java/io/debezium/platform/ServiceTest.java @@ -0,0 +1,202 @@ +package io.debezium.platform; + +import com.blazebit.persistence.view.EntityViewManager; +import io.debezium.platform.domain.DestinationService; +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.SourceService; +import io.debezium.platform.domain.TransformService; +import io.debezium.platform.domain.VaultService; +import io.debezium.platform.domain.views.Destination; +import io.debezium.platform.domain.views.Pipeline; +import io.debezium.platform.domain.views.Source; +import io.debezium.platform.domain.views.Transform; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.domain.views.refs.DestinationReference; +import io.debezium.platform.domain.views.refs.SourceReference; +import io.debezium.platform.domain.views.refs.VaultReference; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import org.assertj.core.api.Assertions; +import org.assertj.core.api.Assumptions; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@QuarkusTest +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ServiceTest { + @Inject + SourceService sourceService; + + @Inject + PipelineService pipelineService; + @Inject + DestinationService destinationService; + @Inject + VaultService vaultService; + @Inject + TransformService transformService; + + @Inject + EntityViewManager evm; + + static List sources = new ArrayList<>(); + static List destinations = new ArrayList<>(); + static List vaults = new ArrayList<>(); + static List transforms = new ArrayList<>(); + static Pipeline pipeline; + + @Test + @Order(0) + public void createVault() { + var vault1 = evm.create(Vault.class); + vault1.setName("vault1"); + vault1.setKeys(List.of("foo", "bar")); + var vault2 = evm.create(Vault.class); + vault2.setName("vault2"); + vault2.setKeys(List.of("baz", "qux")); + + vaults.add(vaultService.create(vault1)); + vaults.add(vaultService.create(vault2)); + } + + @Test + @Order(10) + public void updateVault() { + var v = vaults.get(0); + v.setKeys(List.of("bar")); + v = vaultService.update(v); + vaults.set(0, v); + } + + @Test + @Order(20) + public void createSources() { + Assumptions.assumeThat(vaults).hasSize(2); + + var source1 = evm.create(Source.class); + source1.setName("source1"); + source1.setSchema("schemaXY"); + source1.setType("io.debezium.connector.MongoDbConnector"); + source1.setVaults(List.of(vaultRef(1))); + source1.setConfig(Map.of("mongodb.connection.string", "mongodb://localhost:27017")); + + var source2 = evm.create(Source.class); + source2.setName("source2"); + source2.setSchema("schemaXY"); + source2.setType("io.debezium.connector.MongoDbConnector"); + source2.setVaults(List.of(vaultRef(0))); + source2.setConfig(Map.of("mongodb.connection.string", "mongodb://localhost:37017")); + + sources.add(sourceService.create(source1)); + sources.add(sourceService.create(source2)); + } + + @Test + @Order(30) + public void createDestinations() { + Assumptions.assumeThat(vaults).hasSize(2); + + var destination1 = evm.create(Destination.class); + destination1.setName("destination2"); + destination1.setSchema("schemaDXY"); + destination1.setType("pubsub"); + destination1.setVaults(List.of(vaultRef(0),vaultRef(1))); + destination1.setConfig(Map.of("foo", "bar")); + + var destination2 = evm.create(Destination.class); + destination2.setName("destination2"); + destination2.setSchema("schemaDYZ"); + destination2.setType("redis"); + destination2.setVaults(List.of(vaultRef(0))); + destination2.setConfig(Map.of("bar", "baz")); + + destinations.add(destinationService.create(destination1)); + destinations.add(destinationService.create(destination2)); + } + + @Test + @Order(40) + public void createTransform() { + Assumptions.assumeThat(vaults).hasSize(2); + + var transform1 = evm.create(Transform.class); + transform1.setName("transform1"); + transform1.setSchema("schemaASD"); + transform1.setType("io.example.SomeTransform"); + transform1.setVaults(List.of(vaultRef(0),vaultRef(1))); + transform1.setConfig(Map.of("baz", "qux")); + + transforms.add(transformService.create(transform1)); + } + + @Test + @Order(50) + public void createPipeline() { + Assumptions.assumeThat(sources).isNotEmpty(); + Assumptions.assumeThat(destinations).isNotEmpty(); + Assumptions.assumeThat(transforms).isNotEmpty(); + + var sourceRef = sourceRef(0); + var destinationRef = destinationRef(0); + + var newPipeline = evm.create(Pipeline.class); + newPipeline.setName("pipeline1"); + newPipeline.setSource(sourceRef); + newPipeline.setDestination(destinationRef); + newPipeline.setLogLevel("info"); + + pipeline = pipelineService.create(newPipeline); + } + + @Test + @Order(51) + public void updatePipeline() { + Assumptions.assumeThat(sources).hasSize(2); + Assumptions.assumeThat(destinations).hasSize(2); + Assumptions.assumeThat(pipeline).isNotNull(); + + + var sourceRef = sourceRef(1); + var destinationRef = destinationRef(1); + var uPipeline = pipelineService.findById(pipeline.getId()).orElse(null); + uPipeline.setSource(sourceRef); + uPipeline.setDestination(destinationRef); + + pipeline = pipelineService.update(uPipeline); + } + + @Test + @Order(52) + public void listPipelines() { + var list = pipelineService.list(); + + Assertions.assertThat(list).hasSize(1); + } + + @Test + @Order(100) + public void foo() { + System.out.println(); + } + + public VaultReference vaultRef(int idx) { + var vault = vaults.get(idx); + return vaultService.findReferenceById(vault.getId()).orElseThrow(); + } + + public SourceReference sourceRef(int idx) { + var source = sources.get(idx); + return sourceService.findReferenceById(source.getId()).orElseThrow(); + } + + public DestinationReference destinationRef(int idx) { + var destination = destinations.get(idx); + return destinationService.findReferenceById(destination.getId()).orElseThrow(); + } +} From e26b6214596011a29d8be77b4841bfbbee63d4cf Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 7 May 2024 13:49:29 +0200 Subject: [PATCH 03/44] Initial CRUD API endpoints --- .../platform/api/ConductorApiApplication.java | 8 ++ .../platform/api/DestinationResource.java | 112 ++++++++++++++++++ .../platform/api/PipelineResource.java | 110 +++++++++++++++++ .../debezium/platform/api/SourceResource.java | 112 ++++++++++++++++++ .../platform/api/TransformResource.java | 112 ++++++++++++++++++ .../debezium/platform/api/VaultResource.java | 110 +++++++++++++++++ 6 files changed, 564 insertions(+) create mode 100644 src/main/java/io/debezium/platform/api/ConductorApiApplication.java create mode 100644 src/main/java/io/debezium/platform/api/DestinationResource.java create mode 100644 src/main/java/io/debezium/platform/api/PipelineResource.java create mode 100644 src/main/java/io/debezium/platform/api/SourceResource.java create mode 100644 src/main/java/io/debezium/platform/api/TransformResource.java create mode 100644 src/main/java/io/debezium/platform/api/VaultResource.java diff --git a/src/main/java/io/debezium/platform/api/ConductorApiApplication.java b/src/main/java/io/debezium/platform/api/ConductorApiApplication.java new file mode 100644 index 00000000..4e4bf83a --- /dev/null +++ b/src/main/java/io/debezium/platform/api/ConductorApiApplication.java @@ -0,0 +1,8 @@ +package io.debezium.platform.api; + +import jakarta.ws.rs.ApplicationPath; +import jakarta.ws.rs.core.Application; + +@ApplicationPath("/api") +public class ConductorApiApplication extends Application { +} diff --git a/src/main/java/io/debezium/platform/api/DestinationResource.java b/src/main/java/io/debezium/platform/api/DestinationResource.java new file mode 100644 index 00000000..ab8ba718 --- /dev/null +++ b/src/main/java/io/debezium/platform/api/DestinationResource.java @@ -0,0 +1,112 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.DestinationService; +import io.debezium.platform.domain.views.Destination; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; +import java.util.Objects; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "destinations") +@OpenAPIDefinition( + info = @Info( + title = "Destination API", + description = "CRUD operations over Destination resource", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/api/destinations") +public class DestinationResource { + + Logger logger; + DestinationService destinationService; + + public DestinationResource(Logger logger, DestinationService destinationService) { + this.logger = logger; + this.destinationService = destinationService; + } + + @Operation(summary = "Returns all available destinations") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Destination.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var destinations = destinationService.list(); + return Response.ok(destinations).build(); + } + + @Operation(summary = "Returns a destination with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Destination.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return destinationService.findById(id) + .map(destination -> Response.ok(destination).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new destination") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Destination destination, @Context UriInfo uriInfo) { + var created = destinationService.create(destination); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing destination") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Destination.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Destination destination) { + var updated = destinationService.update(destination); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing destination") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + destinationService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/api/PipelineResource.java b/src/main/java/io/debezium/platform/api/PipelineResource.java new file mode 100644 index 00000000..3f53a8bd --- /dev/null +++ b/src/main/java/io/debezium/platform/api/PipelineResource.java @@ -0,0 +1,110 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.views.Pipeline; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "pipelines") +@OpenAPIDefinition( + info = @Info( + title = "Pipeline API", + description = "CRUD operations over Pipeline resource", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/api/pipelines") +public class PipelineResource { + + Logger logger; + PipelineService pipelineService; + + public PipelineResource(Logger logger, PipelineService pipelineService) { + this.logger = logger; + this.pipelineService = pipelineService; + } + + @Operation(summary = "Returns all available pipelines") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Pipeline.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var pipelines = pipelineService.list(); + return Response.ok(pipelines).build(); + } + + @Operation(summary = "Returns a pipeline with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Pipeline.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return pipelineService.findById(id) + .map(source -> Response.ok(source).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new pipeline") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Pipeline pipeline, @Context UriInfo uriInfo) { + var created = pipelineService.create(pipeline); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing pipeline") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Pipeline.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Pipeline pipeline) { + var updated = pipelineService.update(pipeline); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing pipeline") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + pipelineService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/api/SourceResource.java b/src/main/java/io/debezium/platform/api/SourceResource.java new file mode 100644 index 00000000..38250a6c --- /dev/null +++ b/src/main/java/io/debezium/platform/api/SourceResource.java @@ -0,0 +1,112 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.SourceService; +import io.debezium.platform.domain.views.Source; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; +import java.util.Objects; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "sources") +@OpenAPIDefinition( + info = @Info( + title = "Source API", + description = "CRUD operations over Source resource", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/api/sources") +public class SourceResource { + + Logger logger; + SourceService sourceService; + + public SourceResource(Logger logger, SourceService sourceService) { + this.logger = logger; + this.sourceService = sourceService; + } + + @Operation(summary = "Returns all available sources") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Source.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var sources = sourceService.list(); + return Response.ok(sources).build(); + } + + @Operation(summary = "Returns a source with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Source.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return sourceService.findById(id) + .map(source -> Response.ok(source).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new source") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Source source, @Context UriInfo uriInfo) { + var created = sourceService.create(source); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing source") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Source.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Source source) { + var updated = sourceService.update(source); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing source") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + sourceService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/api/TransformResource.java b/src/main/java/io/debezium/platform/api/TransformResource.java new file mode 100644 index 00000000..09abba88 --- /dev/null +++ b/src/main/java/io/debezium/platform/api/TransformResource.java @@ -0,0 +1,112 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.TransformService; +import io.debezium.platform.domain.views.Transform; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; +import java.util.Objects; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "transforms") +@OpenAPIDefinition( + info = @Info( + title = "Transform API", + description = "CRUD operations over Source revault", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/api/transforms") +public class TransformResource { + + Logger logger; + TransformService transformService; + + public TransformResource(Logger logger, TransformService transformService) { + this.logger = logger; + this.transformService = transformService; + } + + @Operation(summary = "Returns all available vaults") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Transform.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var vaults = transformService.list(); + return Response.ok(vaults).build(); + } + + @Operation(summary = "Returns a transform with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Transform.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return transformService.findById(id) + .map(transform -> Response.ok(transform).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new transform") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Transform transform, @Context UriInfo uriInfo) { + var created = transformService.create(transform); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing transform") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Transform.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Transform transform) { + var updated = transformService.update(transform); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing transform") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + transformService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} diff --git a/src/main/java/io/debezium/platform/api/VaultResource.java b/src/main/java/io/debezium/platform/api/VaultResource.java new file mode 100644 index 00000000..aac331dc --- /dev/null +++ b/src/main/java/io/debezium/platform/api/VaultResource.java @@ -0,0 +1,110 @@ +package io.debezium.platform.api; + +import com.blazebit.persistence.integration.jaxrs.EntityViewId; +import io.debezium.platform.domain.VaultService; +import io.debezium.platform.domain.views.Vault; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; +import org.eclipse.microprofile.openapi.annotations.OpenAPIDefinition; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.info.Contact; +import org.eclipse.microprofile.openapi.annotations.info.Info; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.jboss.logging.Logger; + +import java.net.URI; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Tag(name = "vaults") +@OpenAPIDefinition( + info = @Info( + title = "Vault API", + description = "CRUD operations over Source revault", + version = "0.1.0", + contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") + ) +) +@Path("/api/vaults") +public class VaultResource { + + Logger logger; + VaultService vaultService; + + public VaultResource(Logger logger, VaultService vaultService) { + this.logger = logger; + this.vaultService = vaultService; + } + + @Operation(summary = "Returns all available vaults") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Vault.class, required = true, type = SchemaType.ARRAY)) + ) + @GET + public Response get() { + var vaults = vaultService.list(); + return Response.ok(vaults).build(); + } + + @Operation(summary = "Returns a vault with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Vault.class, required = true)) + ) + @GET + @Path("/{id}") + public Response getById(@PathParam("id") Long id) { + return vaultService.findById(id) + .map(vault -> Response.ok(vault).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } + + @Operation(summary = "Creates new vault") + @APIResponse( + responseCode = "201", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = URI.class, required = true)) + ) + @POST + public Response post(@NotNull @Valid Vault vault, @Context UriInfo uriInfo) { + var created = vaultService.create(vault); + URI uri = uriInfo.getAbsolutePathBuilder() + .path(Long.toString(created.getId())) + .build(); + return Response.created(uri).entity(created).build(); + } + + @Operation(summary = "Updates an existing vault") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = APPLICATION_JSON, schema = @Schema(implementation = Vault.class, required = true)) + ) + @PUT + @Path("/{id}") + public Response put(@EntityViewId("id") @NotNull @Valid Vault vault) { + var updated = vaultService.update(vault); + return Response.ok(updated).build(); + } + + @Operation(summary = "Deletes an existing vault") + @APIResponse(responseCode = "204") + @DELETE + @Path("/{id}") + public Response delete(@PathParam("id") Long id) { + vaultService.delete(id); + return Response.status(Response.Status.NO_CONTENT).build(); + } +} From 85bd114a401936277c7d4abed6befd6e998d66b1 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 16:00:59 +0200 Subject: [PATCH 04/44] Removed blaze persistence processor --- pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pom.xml b/pom.xml index a8cf2294..20428913 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,6 @@ io.quarkus quarkus-smallrye-openapi - com.blazebit blaze-persistence-integration-quarkus-3 @@ -86,13 +85,6 @@ blaze-persistence-integration-jaxrs-jackson-jakarta 1.6.11 - - - - - - - org.projectlombok lombok From 566f2dd83921a14aa01be5c023a0d98b11033577 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 16:02:59 +0200 Subject: [PATCH 05/44] Fixed API base path --- src/main/java/io/debezium/platform/api/DestinationResource.java | 2 +- src/main/java/io/debezium/platform/api/PipelineResource.java | 2 +- src/main/java/io/debezium/platform/api/SourceResource.java | 2 +- src/main/java/io/debezium/platform/api/TransformResource.java | 2 +- src/main/java/io/debezium/platform/api/VaultResource.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/debezium/platform/api/DestinationResource.java b/src/main/java/io/debezium/platform/api/DestinationResource.java index ab8ba718..6ff31b1d 100644 --- a/src/main/java/io/debezium/platform/api/DestinationResource.java +++ b/src/main/java/io/debezium/platform/api/DestinationResource.java @@ -40,7 +40,7 @@ contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") ) ) -@Path("/api/destinations") +@Path("/destinations") public class DestinationResource { Logger logger; diff --git a/src/main/java/io/debezium/platform/api/PipelineResource.java b/src/main/java/io/debezium/platform/api/PipelineResource.java index 3f53a8bd..e7ca4e83 100644 --- a/src/main/java/io/debezium/platform/api/PipelineResource.java +++ b/src/main/java/io/debezium/platform/api/PipelineResource.java @@ -38,7 +38,7 @@ contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") ) ) -@Path("/api/pipelines") +@Path("/pipelines") public class PipelineResource { Logger logger; diff --git a/src/main/java/io/debezium/platform/api/SourceResource.java b/src/main/java/io/debezium/platform/api/SourceResource.java index 38250a6c..25aa42c0 100644 --- a/src/main/java/io/debezium/platform/api/SourceResource.java +++ b/src/main/java/io/debezium/platform/api/SourceResource.java @@ -40,7 +40,7 @@ contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") ) ) -@Path("/api/sources") +@Path("/sources") public class SourceResource { Logger logger; diff --git a/src/main/java/io/debezium/platform/api/TransformResource.java b/src/main/java/io/debezium/platform/api/TransformResource.java index 09abba88..c3fa4427 100644 --- a/src/main/java/io/debezium/platform/api/TransformResource.java +++ b/src/main/java/io/debezium/platform/api/TransformResource.java @@ -40,7 +40,7 @@ contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") ) ) -@Path("/api/transforms") +@Path("/transforms") public class TransformResource { Logger logger; diff --git a/src/main/java/io/debezium/platform/api/VaultResource.java b/src/main/java/io/debezium/platform/api/VaultResource.java index aac331dc..1fa85e2e 100644 --- a/src/main/java/io/debezium/platform/api/VaultResource.java +++ b/src/main/java/io/debezium/platform/api/VaultResource.java @@ -38,7 +38,7 @@ contact = @Contact(name = "Debezium", url = "https://github.com/debezium/debezium") ) ) -@Path("/api/vaults") +@Path("/vaults") public class VaultResource { Logger logger; From 163ff6faf639b05f5d102252660998872bef7cf7 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 16:04:11 +0200 Subject: [PATCH 06/44] Fixed entity removal via view --- src/main/java/io/debezium/platform/domain/AbstractService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/io/debezium/platform/domain/AbstractService.java b/src/main/java/io/debezium/platform/domain/AbstractService.java index 66eb3c6b..81f44bfe 100644 --- a/src/main/java/io/debezium/platform/domain/AbstractService.java +++ b/src/main/java/io/debezium/platform/domain/AbstractService.java @@ -68,7 +68,6 @@ public T update(@Valid T view) { } public void delete(long id) { - evm.remove(em, id); + evm.remove(em, viewType, id); } - } From a69033bcfb91079cc59fb03956c73527e47e4a65 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 16:05:28 +0200 Subject: [PATCH 07/44] Storing vault content as JSON instead of element collection --- .../java/io/debezium/platform/data/model/VaultEntity.java | 7 +++---- src/main/java/io/debezium/platform/domain/views/Vault.java | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/debezium/platform/data/model/VaultEntity.java b/src/main/java/io/debezium/platform/data/model/VaultEntity.java index ecac75d0..cceb3fda 100644 --- a/src/main/java/io/debezium/platform/data/model/VaultEntity.java +++ b/src/main/java/io/debezium/platform/data/model/VaultEntity.java @@ -1,13 +1,13 @@ package io.debezium.platform.data.model; -import jakarta.persistence.Column; -import jakarta.persistence.ElementCollection; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; import jakarta.validation.constraints.NotEmpty; import lombok.Getter; import lombok.Setter; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import java.util.LinkedList; import java.util.List; @@ -22,7 +22,6 @@ public class VaultEntity { @NotEmpty private String name; private boolean plaintext = false; - @ElementCollection - @Column(name = "key") + @JdbcTypeCode(SqlTypes.JSON) private List keys = new LinkedList<>(); } diff --git a/src/main/java/io/debezium/platform/domain/views/Vault.java b/src/main/java/io/debezium/platform/domain/views/Vault.java index 9a6dc45e..c726375d 100644 --- a/src/main/java/io/debezium/platform/domain/views/Vault.java +++ b/src/main/java/io/debezium/platform/domain/views/Vault.java @@ -2,6 +2,7 @@ import com.blazebit.persistence.view.CreatableEntityView; import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.MappingSingular; import com.blazebit.persistence.view.UpdatableEntityView; import io.debezium.platform.data.model.VaultEntity; import io.debezium.platform.domain.views.refs.VaultReference; @@ -13,6 +14,7 @@ @UpdatableEntityView public interface Vault extends VaultReference { boolean isPlaintext(); + @MappingSingular List getKeys(); void setName(String name); From 1cfa4800218eecca117d68551b007331cbdad40d Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 16:07:17 +0200 Subject: [PATCH 08/44] Added missing @UpdatableEntityView to Pipeline --- src/main/java/io/debezium/platform/domain/views/Pipeline.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/io/debezium/platform/domain/views/Pipeline.java b/src/main/java/io/debezium/platform/domain/views/Pipeline.java index 9af5042d..7d1a52aa 100644 --- a/src/main/java/io/debezium/platform/domain/views/Pipeline.java +++ b/src/main/java/io/debezium/platform/domain/views/Pipeline.java @@ -2,6 +2,7 @@ import com.blazebit.persistence.view.CreatableEntityView; import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; import io.debezium.platform.data.model.PipelineEntity; import io.debezium.platform.domain.views.base.NamedView; import io.debezium.platform.domain.views.refs.DestinationReference; @@ -14,6 +15,7 @@ @EntityView(PipelineEntity.class) @CreatableEntityView +@UpdatableEntityView public interface Pipeline extends NamedView { @NotNull SourceReference getSource(); From 430a4f4877e0b4c881949133627fd14393ff4e9d Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 16:12:43 +0200 Subject: [PATCH 09/44] Pipeline and Vault CRUD operations now also send an appropriate outbox message --- pom.xml | 11 ++++ .../platform/domain/AbstractService.java | 5 ++ .../platform/domain/PipelineService.java | 51 +++++++++++++++++++ .../platform/domain/VaultService.java | 40 +++++++++++++++ .../domain/views/flat/PipelineFlat.java | 29 +++++++++++ src/main/resources/application.yml | 22 +++++++- 6 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java diff --git a/pom.xml b/pom.xml index 20428913..11114a8d 100644 --- a/pom.xml +++ b/pom.xml @@ -35,9 +35,20 @@ pom import + + ${quarkus.platform.group-id} + quarkus-debezium-bom + ${quarkus.platform.version} + pom + import + + + io.debezium + debezium-quarkus-outbox + io.quarkus quarkus-config-yaml diff --git a/src/main/java/io/debezium/platform/domain/AbstractService.java b/src/main/java/io/debezium/platform/domain/AbstractService.java index 81f44bfe..ba46ecaf 100644 --- a/src/main/java/io/debezium/platform/domain/AbstractService.java +++ b/src/main/java/io/debezium/platform/domain/AbstractService.java @@ -53,6 +53,11 @@ public List list() { @Transactional(SUPPORTS) public Optional findById(Long id) { + return findByIdAs(viewType, id); + } + + @Transactional(SUPPORTS) + public Optional findByIdAs(Class viewType, Long id) { var result = evm.find(em, viewType, id); return Optional.ofNullable(result); } diff --git a/src/main/java/io/debezium/platform/domain/PipelineService.java b/src/main/java/io/debezium/platform/domain/PipelineService.java index beddf2ab..4ad0fdfe 100644 --- a/src/main/java/io/debezium/platform/domain/PipelineService.java +++ b/src/main/java/io/debezium/platform/domain/PipelineService.java @@ -3,15 +3,66 @@ import com.blazebit.persistence.CriteriaBuilderFactory; import com.blazebit.persistence.view.EntityViewManager; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.outbox.quarkus.ExportedEvent; import io.debezium.platform.data.model.PipelineEntity; import io.debezium.platform.domain.views.Pipeline; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.watcher.events.PipelineEvent; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Event; +import jakarta.inject.Inject; import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.Optional; @ApplicationScoped public class PipelineService extends AbstractService { + @Inject + Event> event; + + @Inject + ObjectMapper objectMapper; + public PipelineService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { super(PipelineEntity.class, Pipeline.class, em, cbf, evm); } + + @Transactional(Transactional.TxType.SUPPORTS) + public Optional findByIdFlat(Long id) { + var result = evm.find(em, PipelineFlat.class, id); + return Optional.ofNullable(result); + } + + @Override + public Pipeline create(Pipeline view) { + var result = super.create(view); + fireUpdateEvent(result.getId()); + return result; + } + + @Override + public Pipeline update(Pipeline view) { + var result = super.update(view); + fireUpdateEvent(result.getId()); + return result; + } + + @Override + public void delete(long id) { + super.delete(id); + fireDeleteEvent(id); + } + + private void fireUpdateEvent(Long id) { + var flat = findByIdFlat(id); + flat.ifPresent(pipeline -> event.fire(PipelineEvent.update(pipeline, objectMapper))); + } + + private void fireDeleteEvent(Long id) { + event.fire(PipelineEvent.delete(id)); + } + } diff --git a/src/main/java/io/debezium/platform/domain/VaultService.java b/src/main/java/io/debezium/platform/domain/VaultService.java index 80163b62..ea668625 100644 --- a/src/main/java/io/debezium/platform/domain/VaultService.java +++ b/src/main/java/io/debezium/platform/domain/VaultService.java @@ -3,10 +3,15 @@ import com.blazebit.persistence.CriteriaBuilderFactory; import com.blazebit.persistence.view.EntityViewManager; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.outbox.quarkus.ExportedEvent; import io.debezium.platform.data.model.VaultEntity; import io.debezium.platform.domain.views.Vault; import io.debezium.platform.domain.views.refs.VaultReference; +import io.debezium.platform.environment.watcher.events.VaultEvent; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Event; +import jakarta.inject.Inject; import jakarta.persistence.EntityManager; import jakarta.transaction.Transactional; @@ -18,6 +23,12 @@ @ApplicationScoped public class VaultService extends AbstractService { + @Inject + Event> event; + + @Inject + ObjectMapper objectMapper; + public VaultService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { super(VaultEntity.class, Vault.class, em, cbf, evm); } @@ -27,4 +38,33 @@ public Optional findReferenceById(Long id) { var result = evm.find(em, VaultReference.class, id); return Optional.ofNullable(result); } + + @Override + public Vault create(Vault view) { + var result = super.create(view); + fireUpdateEvent(result.getId()); + return result; + } + + @Override + public Vault update(Vault view) { + var result = super.update(view); + fireUpdateEvent(result.getId()); + return result; + } + + @Override + public void delete(long id) { + super.delete(id); + fireDeleteEvent(id); + } + + private void fireUpdateEvent(Long id) { + var flat = findById(id); + flat.ifPresent(vault -> event.fire(VaultEvent.update(vault, objectMapper))); + } + + private void fireDeleteEvent(Long id) { + event.fire(VaultEvent.delete(id)); + } } diff --git a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java new file mode 100644 index 00000000..9a770a18 --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java @@ -0,0 +1,29 @@ +package io.debezium.platform.domain.views.flat; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.Destination; +import io.debezium.platform.domain.views.Source; +import io.debezium.platform.domain.views.Transform; +import io.debezium.platform.domain.views.base.NamedView; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +import java.util.List; + +@EntityView(PipelineEntity.class) +public interface PipelineFlat extends NamedView { + @NotNull + Source getSource(); + @NotNull + Destination getDestination(); + List getTransforms(); + @NotEmpty + String getLogLevel(); + + void setName(String name); + void setSource(Source source); + void setDestination(Destination destination); + void setLogLevel(String logLevel); + void setTransforms(List transforms); +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index f1f01031..88d72f82 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,26 @@ quarkus: + debezium-outbox: + table-name: events + aggregate-type: + name: aggregatetype + aggregate-id: + name: aggregateid + type: + name: type datasource: - db-kid: postgresql + db-kind: postgresql devservices: enabled: true port: 5432 + image-name: quay.io/debezium/postgres:14-alpine + log: + level: INFO + +"%dev": + quarkus: + debezium-outbox: + remove-after-insert: false + log: + level: INFO + category: + "io.debezium": DEBUG \ No newline at end of file From 08426e6913ad37457b8c03e7877e259fd0eafac3 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 16:20:04 +0200 Subject: [PATCH 10/44] Initial draft of Environment SPI --- pom.xml | 18 ++++ .../environment/EnvironmentController.java | 8 ++ .../environment/PipelineController.java | 11 +++ .../platform/environment/VaultController.java | 10 ++ .../watcher/ConductorEnvironmentWatcher.java | 98 +++++++++++++++++++ .../watcher/config/DataSourceConfigGroup.java | 13 +++ .../watcher/config/OutboxConfigGroup.java | 20 ++++ .../watcher/config/WatcherConfig.java | 41 ++++++++ .../watcher/config/WatcherConfigGroup.java | 20 ++++ .../consumers/AbstractEventConsumer.java | 38 +++++++ .../consumers/EnvironmentEventConsumer.java | 74 ++++++++++++++ .../consumers/OutboxParentEventConsumer.java | 49 ++++++++++ .../watcher/consumers/PipelineConsumer.java | 39 ++++++++ .../watcher/consumers/VaultConsumer.java | 44 +++++++++ .../watcher/events/AbstractEvent.java | 33 +++++++ .../environment/watcher/events/EventType.java | 6 ++ .../watcher/events/PipelineEvent.java | 25 +++++ .../watcher/events/VaultEvent.java | 25 +++++ src/main/resources/application.yml | 14 +++ 19 files changed, 586 insertions(+) create mode 100644 src/main/java/io/debezium/platform/environment/EnvironmentController.java create mode 100644 src/main/java/io/debezium/platform/environment/PipelineController.java create mode 100644 src/main/java/io/debezium/platform/environment/VaultController.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/ConductorEnvironmentWatcher.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/config/DataSourceConfigGroup.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/config/OutboxConfigGroup.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfig.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfigGroup.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/consumers/AbstractEventConsumer.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/consumers/EnvironmentEventConsumer.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/consumers/OutboxParentEventConsumer.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/consumers/PipelineConsumer.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/consumers/VaultConsumer.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/events/AbstractEvent.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/events/EventType.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/events/PipelineEvent.java create mode 100644 src/main/java/io/debezium/platform/environment/watcher/events/VaultEvent.java diff --git a/pom.xml b/pom.xml index 11114a8d..90e9f38b 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,24 @@ io.debezium debezium-quarkus-outbox + + io.debezium + debezium-api + + + io.debezium + debezium-embedded + + + io.debezium + debezium-connector-postgres + + + io.debezium + debezium-operator-api + + 2.6.1.Final + io.quarkus quarkus-config-yaml diff --git a/src/main/java/io/debezium/platform/environment/EnvironmentController.java b/src/main/java/io/debezium/platform/environment/EnvironmentController.java new file mode 100644 index 00000000..e1d3ab60 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/EnvironmentController.java @@ -0,0 +1,8 @@ +package io.debezium.platform.environment; + +public interface EnvironmentController { + + PipelineController pipelines(); + VaultController vaults(); + +} diff --git a/src/main/java/io/debezium/platform/environment/PipelineController.java b/src/main/java/io/debezium/platform/environment/PipelineController.java new file mode 100644 index 00000000..ab1d37ed --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/PipelineController.java @@ -0,0 +1,11 @@ +package io.debezium.platform.environment; + +import io.debezium.platform.domain.views.flat.PipelineFlat; + +public interface PipelineController { + + void deploy(PipelineFlat pipeline); + + void undeploy(Long id); + +} diff --git a/src/main/java/io/debezium/platform/environment/VaultController.java b/src/main/java/io/debezium/platform/environment/VaultController.java new file mode 100644 index 00000000..6ae46aeb --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/VaultController.java @@ -0,0 +1,10 @@ +package io.debezium.platform.environment; + +import io.debezium.platform.domain.views.Vault; + +public interface VaultController { + + void deploy(Vault vault); + + void undeploy(Long id); +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/ConductorEnvironmentWatcher.java b/src/main/java/io/debezium/platform/environment/watcher/ConductorEnvironmentWatcher.java new file mode 100644 index 00000000..f0be55ed --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/ConductorEnvironmentWatcher.java @@ -0,0 +1,98 @@ +package io.debezium.platform.environment.watcher; + +import io.debezium.config.Configuration; +import io.debezium.connector.postgresql.PostgresConnector; +import io.debezium.connector.postgresql.PostgresConnectorConfig; +import io.debezium.embedded.Connect; +import io.debezium.embedded.EmbeddedEngineConfig; +import io.debezium.engine.DebeziumEngine; +import io.debezium.platform.environment.watcher.config.WatcherConfig; +import io.debezium.platform.environment.watcher.consumers.OutboxParentEventConsumer; +import io.debezium.transforms.outbox.EventRouter; +import io.quarkus.runtime.ShutdownEvent; +import io.quarkus.runtime.Startup; +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import org.jboss.logging.Logger; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@ApplicationScoped +@Startup +public class ConductorEnvironmentWatcher { + + private final Logger logger; + private final OutboxParentEventConsumer eventConsumer; + private final WatcherConfig watcherConfig; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private DebeziumEngine engine; + + public ConductorEnvironmentWatcher(Logger logger, WatcherConfig watcherConfig, OutboxParentEventConsumer eventConsumer) { + this.logger = logger; + this.watcherConfig = watcherConfig; + this.eventConsumer = eventConsumer; + } + + @PostConstruct + public void start() { + if (!watcherConfig.watcher().enabled()) { + logger.info("Skipping watcher because it is not enabled"); + return; + } + + var connection = watcherConfig.connection(); + var offset = watcherConfig.watcher().offset(); + var outbox = watcherConfig.outbox(); + var extraFields = Stream.of(outbox.aggregateColumn(), outbox.aggregateIdColumn(), outbox.typeColumn()) + .map(c -> c + ":envelope") + .collect(Collectors.joining(",")); + + var config = Configuration.create() + .with(EmbeddedEngineConfig.ENGINE_NAME, "conductor") + .with(EmbeddedEngineConfig.CONNECTOR_CLASS, PostgresConnector.class.getName()) + .with(EmbeddedEngineConfig.OFFSET_STORAGE, offset.storage()) + .with(EmbeddedEngineConfig.OFFSET_STORAGE_FILE_FILENAME, offset.file()) + .with(PostgresConnectorConfig.TOPIC_PREFIX, "conductor") + .with(PostgresConnectorConfig.HOSTNAME, connection.host()) + .with(PostgresConnectorConfig.PORT, connection.port()) + .with(PostgresConnectorConfig.USER, connection.username()) + .with(PostgresConnectorConfig.PASSWORD, connection.password()) + .with(PostgresConnectorConfig.DATABASE_NAME, connection.database()) + .with(PostgresConnectorConfig.PLUGIN_NAME, PostgresConnectorConfig.LogicalDecoder.PGOUTPUT.getValue()) + .with(PostgresConnectorConfig.INCLUDE_SCHEMA_CHANGES, false) + .with(PostgresConnectorConfig.TABLE_INCLUDE_LIST, "public.%s".formatted(outbox.table())) + .with("transforms", "outbox") + .with("transforms.outbox.type", EventRouter.class.getName()) + .with("transforms.outbox.table.fields.additional.placement", extraFields) + .build(); + + logger.info("Creating Debezium engine"); + this.engine = DebeziumEngine.create(Connect.class) + .using(config.asProperties()) + .notifying(eventConsumer) + .build(); + + logger.info("Attempting to start debezium engine"); + executor.execute(engine); + } + + public void stop(@Observes ShutdownEvent event) { + if (engine == null) { + return; + } + + try { + logger.info("Attempting to stop Debezium"); + engine.close(); + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.SECONDS); + } catch (Exception e) { + logger.error("Exception while shutting down Debezium", e); + } + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/DataSourceConfigGroup.java b/src/main/java/io/debezium/platform/environment/watcher/config/DataSourceConfigGroup.java new file mode 100644 index 00000000..136ea0d1 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/DataSourceConfigGroup.java @@ -0,0 +1,13 @@ +package io.debezium.platform.environment.watcher.config; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; + +@ConfigMapping(prefix = "quarkus.datasource") +public interface DataSourceConfigGroup { + + String username(); + String password(); + @WithName("jdbc.url") + String url(); +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/OutboxConfigGroup.java b/src/main/java/io/debezium/platform/environment/watcher/config/OutboxConfigGroup.java new file mode 100644 index 00000000..bb4127ca --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/OutboxConfigGroup.java @@ -0,0 +1,20 @@ +package io.debezium.platform.environment.watcher.config; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; + +@ConfigMapping(prefix = "quarkus.debezium-outbox") +public interface OutboxConfigGroup { + + @WithName("table-name") + String table(); + + @WithName("aggregate-type.name") + String aggregateColumn(); + + @WithName("aggregate-id.name") + String aggregateIdColumn(); + + @WithName("type.name") + String typeColumn(); +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfig.java b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfig.java new file mode 100644 index 00000000..35778845 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfig.java @@ -0,0 +1,41 @@ +package io.debezium.platform.environment.watcher.config; + +import jakarta.enterprise.context.Dependent; +import lombok.Getter; +import lombok.experimental.Accessors; + +@Dependent +@Accessors(fluent = true) +@Getter +public final class WatcherConfig { + + private final WatcherConfigGroup watcher; + private final OutboxConfigGroup outbox; + private final ConnectionConfig connection; + + public record ConnectionConfig(String username, String password, String database, String host, int port) {} + + public WatcherConfig(DataSourceConfigGroup dsConfig, WatcherConfigGroup watcher, OutboxConfigGroup outbox) { + this.watcher = watcher; + this.outbox = outbox; + this.connection = createConnectionConfig(dsConfig); + } + + private static ConnectionConfig createConnectionConfig(DataSourceConfigGroup dsConfig) { + var url = dsConfig.url(); + var hostStart = url.indexOf("://") + 3; + var hostEnd = url.indexOf(":", hostStart); + var portStart = hostEnd + 1; + var portEnd = url.indexOf("/", portStart); + var databaseStart = portEnd + 1; + var databaseEnd = url.indexOf("?", databaseStart); + + var username = dsConfig.username(); + var password = dsConfig.password(); + var database = url.substring(databaseStart, (databaseEnd != -1)? databaseEnd : url.length()); + var host = url.substring(hostStart, hostEnd); + var port = url.substring(portStart, portEnd); + + return new ConnectionConfig(username, password, database, host, Integer.parseInt(port)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfigGroup.java b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfigGroup.java new file mode 100644 index 00000000..82078997 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/config/WatcherConfigGroup.java @@ -0,0 +1,20 @@ +package io.debezium.platform.environment.watcher.config; + +import io.smallrye.config.ConfigMapping; + +import java.util.Optional; + +@ConfigMapping(prefix = "conductor.watcher") +public interface WatcherConfigGroup { + + boolean enabled(); + + Optional crd(); + + OffsetConfigGroup offset(); + + interface OffsetConfigGroup { + String storage(); + String file(); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/AbstractEventConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/AbstractEventConsumer.java new file mode 100644 index 00000000..31ec8103 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/AbstractEventConsumer.java @@ -0,0 +1,38 @@ +package io.debezium.platform.environment.watcher.consumers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.environment.EnvironmentController; +import org.jboss.logging.Logger; + +public abstract class AbstractEventConsumer implements EnvironmentEventConsumer { + + protected final Logger logger; + protected final EnvironmentController environment; + protected final ObjectMapper objectMapper; + protected final Class payloadType; + + public AbstractEventConsumer(Logger logger, EnvironmentController environment, ObjectMapper objectMapper, Class payloadType) { + this.logger = logger; + this.environment = environment; + this.objectMapper = objectMapper; + this.payloadType = payloadType; + } + + @Override + public Class consumedPayloadType() { + return payloadType; + } + + @Override + public T convert(String payload) { + if (payload == null) { + return null; + } + try { + return objectMapper.readValue(payload, consumedPayloadType()); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/EnvironmentEventConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/EnvironmentEventConsumer.java new file mode 100644 index 00000000..0c4f9700 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/EnvironmentEventConsumer.java @@ -0,0 +1,74 @@ +package io.debezium.platform.environment.watcher.consumers; + +import java.util.Collection; +import java.util.Optional; +import java.util.function.BiConsumer; + + +/** + * Event consumers + * @param payload type + */ +public interface EnvironmentEventConsumer extends BiConsumer> { + + /** + * @return collection of consumable aggregate types + */ + Collection consumedAggregates(); + + /** + * @return collection of consumable event types + */ + Collection consumedTypes(); + + /** + * @return consumed payload class + */ + Class consumedPayloadType(); + + /** + * Determines whether this consumer consumes events for given aggregate + * and event types. By default, empty list returned by + * {@link #consumedAggregates()} or {@link #consumedTypes()} is + * treated as a wildcard. + * + * @param aggregateType event aggregate eventType + * @param eventType event eventType + * @return true if the event should be consumed false otherwise + */ + default boolean consumes(String aggregateType, String eventType) { + var aggregates = consumedAggregates(); + var types = consumedTypes(); + + if (!aggregates.isEmpty() && !aggregates.contains(aggregateType)) { + return false; + } + + return types.isEmpty() || types.contains(eventType); + } + + /** + * Converts json payload into object accepted by this consumer + * @param payload json payload + * + * @return converted payload object + */ + T convert(String payload); + + /** + * A shortcut method which takes event aggregate,type and payload. + * Determines whether this consumer is applicable and if so, converts the payload and + * calls {@link #accept(Object, Object)} )}. + * + * @param aggregateType event aggregate type + * @param eventType event type + * @param id aggregate id + * @param payload json payload + */ + default void consume(String aggregateType, String eventType, Long id, String payload) { + if (consumes(aggregateType, eventType)) { + var object = convert(payload); + accept(id, Optional.ofNullable(object)); + } + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/OutboxParentEventConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/OutboxParentEventConsumer.java new file mode 100644 index 00000000..89926e89 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/OutboxParentEventConsumer.java @@ -0,0 +1,49 @@ +package io.debezium.platform.environment.watcher.consumers; + +import io.debezium.engine.ChangeEvent; +import io.debezium.platform.environment.watcher.config.OutboxConfigGroup; +import jakarta.enterprise.context.Dependent; +import jakarta.enterprise.inject.Instance; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.jboss.logging.Logger; + +import java.util.function.Consumer; + +/** + * Top level consumer of outbox events. Parent consumer will extract + * required information from the accepted {@link ChangeEvent} and delegate + * to all registered instances of {@link EnvironmentEventConsumer} + *
+ * + * It's then up to {@link EnvironmentEventConsumer} instances to either process + * or ignore the event. + */ +@Dependent +public final class OutboxParentEventConsumer implements Consumer> { + + private final Logger logger; + private final OutboxConfigGroup outbox; + private final Instance> eventConsumers; + + public OutboxParentEventConsumer(Logger logger, OutboxConfigGroup outbox, Instance> eventConsumers) { + this.logger = logger; + this.outbox = outbox; + this.eventConsumers = eventConsumers; + } + + @Override + public void accept(ChangeEvent event) { + var value = (Struct) event.value().value(); + + var aggregateType = value.getString(outbox.aggregateColumn()); + var aggregateId = value.getString(outbox.aggregateIdColumn()); + var eventType = value.getString(outbox.typeColumn()); + var payload = value.getString("payload"); + + logger.debugf("Consumed %s event for % (#%d)", eventType, aggregateType, aggregateId); + + eventConsumers.forEach(consumer -> consumer.consume( + aggregateType, eventType, Long.valueOf(aggregateId), payload)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/PipelineConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/PipelineConsumer.java new file mode 100644 index 00000000..1004cf8d --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/PipelineConsumer.java @@ -0,0 +1,39 @@ +package io.debezium.platform.environment.watcher.consumers; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.watcher.events.EventType; +import jakarta.enterprise.context.Dependent; +import org.jboss.logging.Logger; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Dependent +public class PipelineConsumer extends AbstractEventConsumer { + + public PipelineConsumer(Logger logger, EnvironmentController environment, ObjectMapper objectMapper) { + super(logger, environment, objectMapper, PipelineFlat.class); + } + + @Override + public Collection consumedAggregates() { + return List.of("pipeline"); + } + + @Override + public Collection consumedTypes() { + return List.of(EventType.UPDATE.name(), EventType.DELETE.name()); + } + + @Override + public void accept(Long id, Optional payload) { + logger.info("Received pipeline event: " + id); + logger.info(">>> payload: \n" + payload); + var pipelines = environment.pipelines(); + + payload.ifPresentOrElse(pipelines::deploy, () -> pipelines.undeploy(id)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/consumers/VaultConsumer.java b/src/main/java/io/debezium/platform/environment/watcher/consumers/VaultConsumer.java new file mode 100644 index 00000000..b59d658e --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/consumers/VaultConsumer.java @@ -0,0 +1,44 @@ +package io.debezium.platform.environment.watcher.consumers; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.watcher.events.EventType; +import jakarta.enterprise.context.Dependent; +import org.jboss.logging.Logger; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +@Dependent +public class VaultConsumer extends AbstractEventConsumer { + + public VaultConsumer(Logger logger, EnvironmentController environment, ObjectMapper objectMapper) { + super(logger, environment, objectMapper, Vault.class); + } + + @Override + public Collection consumedAggregates() { + return List.of("vault"); + } + + @Override + public Collection consumedTypes() { + return List.of(EventType.UPDATE.name(), EventType.DELETE.name()); + } + + @Override + public Class consumedPayloadType() { + return Vault.class; + } + + @Override + public void accept(Long id, Optional payload) { + logger.info("Received vault event: " + id); + logger.info(">>> payload: \n" + payload); + var vaults = environment.vaults(); + + payload.ifPresentOrElse(vaults::deploy, () -> vaults.undeploy(id)); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/AbstractEvent.java b/src/main/java/io/debezium/platform/environment/watcher/events/AbstractEvent.java new file mode 100644 index 00000000..89d09ccf --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/AbstractEvent.java @@ -0,0 +1,33 @@ +package io.debezium.platform.environment.watcher.events; + +import com.fasterxml.jackson.databind.JsonNode; +import io.debezium.outbox.quarkus.ExportedEvent; +import lombok.Getter; +import lombok.Setter; + +import java.time.Instant; + +@Getter +@Setter +public abstract class AbstractEvent + implements ExportedEvent { + + private final String aggregateType; + private final String aggregateId; + private final String type; + private final JsonNode payload; + private final Instant timestamp; + + public AbstractEvent( + String aggregateType, + String aggregateId, + EventType type, + Instant timestamp, + JsonNode payload) { + this.aggregateType = aggregateType; + this.aggregateId = aggregateId; + this.type = type.name(); + this.payload = payload; + this.timestamp = timestamp; + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/EventType.java b/src/main/java/io/debezium/platform/environment/watcher/events/EventType.java new file mode 100644 index 00000000..b6228288 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/EventType.java @@ -0,0 +1,6 @@ +package io.debezium.platform.environment.watcher.events; + +public enum EventType { + UPDATE, + DELETE +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/PipelineEvent.java b/src/main/java/io/debezium/platform/environment/watcher/events/PipelineEvent.java new file mode 100644 index 00000000..fa855f0f --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/PipelineEvent.java @@ -0,0 +1,25 @@ +package io.debezium.platform.environment.watcher.events; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.flat.PipelineFlat; + +import java.time.Instant; + +public final class PipelineEvent extends AbstractEvent { + + private static final String AGGREGATE_TYPE = "pipeline"; + + private PipelineEvent(String aggregateId, EventType type, Instant timestamp, JsonNode payload) { + super(AGGREGATE_TYPE, aggregateId, type, timestamp, payload); + } + + public static PipelineEvent update(PipelineFlat pipeline, ObjectMapper objectMapper) { + var payload = objectMapper.valueToTree(pipeline); + return new PipelineEvent(pipeline.getId().toString(), EventType.UPDATE, Instant.now(), payload); + } + + public static PipelineEvent delete(Long id) { + return new PipelineEvent(id.toString(), EventType.DELETE, Instant.now(), null); + } +} diff --git a/src/main/java/io/debezium/platform/environment/watcher/events/VaultEvent.java b/src/main/java/io/debezium/platform/environment/watcher/events/VaultEvent.java new file mode 100644 index 00000000..753f71f7 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/watcher/events/VaultEvent.java @@ -0,0 +1,25 @@ +package io.debezium.platform.environment.watcher.events; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.platform.domain.views.Vault; + +import java.time.Instant; + +public final class VaultEvent extends AbstractEvent { + + private static final String AGGREGATE_TYPE = "vault"; + + private VaultEvent(String aggregateId, EventType type, Instant timestamp, JsonNode payload) { + super(AGGREGATE_TYPE, aggregateId, type, timestamp, payload); + } + + public static VaultEvent update(Vault vault, ObjectMapper objectMapper) { + var payload = objectMapper.valueToTree(vault); + return new VaultEvent(vault.getId().toString(), EventType.UPDATE, Instant.now(), payload); + } + + public static VaultEvent delete(Long id) { + return new VaultEvent(id.toString(), EventType.DELETE, Instant.now(), null); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 88d72f82..9584417d 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,3 +1,11 @@ +conductor: + watcher: +# For now watcher is disabled in prod as we need only the REST API + enabled: false + offset: + storage: org.apache.kafka.connect.storage.FileOffsetBackingStore + file: offsets.dat + quarkus: debezium-outbox: table-name: events @@ -17,6 +25,12 @@ quarkus: level: INFO "%dev": + conductor: + watcher: + enabled: true + crd: https://raw.githubusercontent.com/debezium/debezium-operator/main/k8/debeziumservers.debezium.io-v1.yml + offset: + storage: org.apache.kafka.connect.storage.MemoryOffsetBackingStore quarkus: debezium-outbox: remove-after-insert: false From 1e205340c215231e7680c6919c397fd55a864e95 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 19:13:34 +0200 Subject: [PATCH 11/44] PipelineFlat must be updatable in order to expose setters for deserialisation --- .../io/debezium/platform/domain/views/flat/PipelineFlat.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java index 9a770a18..e194fa7a 100644 --- a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java +++ b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java @@ -1,6 +1,7 @@ package io.debezium.platform.domain.views.flat; import com.blazebit.persistence.view.EntityView; +import com.blazebit.persistence.view.UpdatableEntityView; import io.debezium.platform.data.model.PipelineEntity; import io.debezium.platform.domain.views.Destination; import io.debezium.platform.domain.views.Source; @@ -12,6 +13,7 @@ import java.util.List; @EntityView(PipelineEntity.class) +@UpdatableEntityView public interface PipelineFlat extends NamedView { @NotNull Source getSource(); From bf25780ea05ab36a18f25e5aa4af5877ae6249e4 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 19:14:09 +0200 Subject: [PATCH 12/44] Initial implementation of operator environment --- pom.xml | 4 + .../environment/operator/DevCrdInstaller.java | 46 +++++++++++ .../OperatorEnvironmentController.java | 34 ++++++++ .../operator/OperatorPipelineController.java | 82 +++++++++++++++++++ .../operator/OperatorVaultController.java | 26 ++++++ 5 files changed, 192 insertions(+) create mode 100644 src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java create mode 100644 src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java create mode 100644 src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java create mode 100644 src/main/java/io/debezium/platform/environment/operator/OperatorVaultController.java diff --git a/pom.xml b/pom.xml index 90e9f38b..79d194d7 100644 --- a/pom.xml +++ b/pom.xml @@ -67,6 +67,10 @@ 2.6.1.Final + + io.quarkus + quarkus-kubernetes-client + io.quarkus quarkus-config-yaml diff --git a/src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java b/src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java new file mode 100644 index 00000000..75da6e91 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java @@ -0,0 +1,46 @@ +package io.debezium.platform.environment.operator; + +import io.debezium.platform.environment.watcher.config.WatcherConfigGroup; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.quarkus.arc.profile.IfBuildProfile; +import io.quarkus.runtime.Startup; +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.jboss.logging.Logger; + +import java.net.MalformedURLException; +import java.net.URI; + +@Startup +@ApplicationScoped +@IfBuildProfile("dev") +public class DevCrdInstaller { + + Logger logger; + KubernetesClient k8s; + WatcherConfigGroup watcherConfig; + + public DevCrdInstaller(Logger logger, KubernetesClient k8s, WatcherConfigGroup watcherConfig) { + this.logger = logger; + this.k8s = k8s; + this.watcherConfig = watcherConfig; + } + + @PostConstruct + public void init() { + watcherConfig.crd().ifPresent(this::install); + } + + public void install(String crdUrl) { + try { + var url = URI.create(crdUrl).toURL(); + var crds = k8s.apiextensions().v1().customResourceDefinitions(); + var crd = crds.load(url).item(); + + crds.resource(crd).serverSideApply(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java new file mode 100644 index 00000000..8669daee --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java @@ -0,0 +1,34 @@ +package io.debezium.platform.environment.operator; + +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.PipelineController; +import io.debezium.platform.environment.VaultController; +import jakarta.enterprise.context.ApplicationScoped; +import org.jboss.logging.Logger; + +@ApplicationScoped +public class OperatorEnvironmentController implements EnvironmentController { + + protected final Logger logger; + private final OperatorPipelineController pipelineController; + private final OperatorVaultController vaultController; + + public OperatorEnvironmentController( + Logger logger, + OperatorPipelineController pipelineController, + OperatorVaultController vaultController) { + this.logger = logger; + this.pipelineController = pipelineController; + this.vaultController = vaultController; + } + + @Override + public PipelineController pipelines() { + return pipelineController; + } + + @Override + public VaultController vaults() { + return vaultController; + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java new file mode 100644 index 00000000..fca5f05c --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java @@ -0,0 +1,82 @@ +package io.debezium.platform.environment.operator; + +import io.debezium.operator.api.model.ConfigProperties; +import io.debezium.operator.api.model.DebeziumServer; +import io.debezium.operator.api.model.DebeziumServerBuilder; +import io.debezium.operator.api.model.DebeziumServerSpecBuilder; +import io.debezium.operator.api.model.QuarkusBuilder; +import io.debezium.operator.api.model.SinkBuilder; +import io.debezium.operator.api.model.SourceBuilder; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.PipelineController; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import jakarta.enterprise.context.Dependent; + +import java.util.Map; + + +@Dependent +public class OperatorPipelineController implements PipelineController { + + public static final String LABEL_DBZ_CONDUCTOR_ID = "debezium.io/conductor-id"; + + private final KubernetesClient k8s; + + public OperatorPipelineController(KubernetesClient k8s) { + this.k8s = k8s; + } + + @Override + public void deploy(PipelineFlat pipeline) { + // Create DS quarkus configuration + var quarkusConfig = new ConfigProperties(); + quarkusConfig.setProps("log.leve", pipeline.getLogLevel()); + var dsQuarkus = new QuarkusBuilder() + .withConfig(quarkusConfig) + .build(); + + // Create DS source configuration + var source = pipeline.getSource(); + var sourceConfig = new ConfigProperties(); + sourceConfig.setAllProps(source.getConfig()); + + var dsSource = new SourceBuilder() + .withSourceClass(source.getType()) + .withConfig(sourceConfig) + .build(); + + // Create DS sink configuration + var sink = pipeline.getDestination(); + var sinkConfig = new ConfigProperties(); + sinkConfig.setAllProps(sink.getConfig()); + + var dsSink = new SinkBuilder() + .withType(sink.getType()) + .withConfig(sinkConfig) + .build(); + + // Create DS resource + var ds = new DebeziumServerBuilder() + .withMetadata(new ObjectMetaBuilder() + .withName(pipeline.getName()) + .withLabels(Map.of(LABEL_DBZ_CONDUCTOR_ID, pipeline.getId().toString())) + .build()) + .withSpec(new DebeziumServerSpecBuilder() + .withQuarkus(dsQuarkus) + .withSource(dsSource) + .withSink(dsSink) + .build()) + .build(); + + // apply to server + k8s.resource(ds).serverSideApply(); + } + + @Override + public void undeploy(Long id) { + k8s.resources(DebeziumServer.class) + .withLabels(Map.of(LABEL_DBZ_CONDUCTOR_ID, id.toString())) + .delete(); + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorVaultController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorVaultController.java new file mode 100644 index 00000000..83e7f750 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorVaultController.java @@ -0,0 +1,26 @@ +package io.debezium.platform.environment.operator; + +import io.debezium.platform.domain.views.Vault; +import io.debezium.platform.environment.VaultController; +import io.fabric8.kubernetes.client.KubernetesClient; +import jakarta.enterprise.context.Dependent; + + +@Dependent +public class OperatorVaultController implements VaultController { + + private final KubernetesClient k8s; + + public OperatorVaultController(KubernetesClient k8s) { + this.k8s = k8s; + } + + @Override + public void deploy(Vault vault) { + } + + @Override + public void undeploy(Long id) { + + } +} From 3857a050a0614699132a85323ee17c4d4ab640be Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 21:55:34 +0200 Subject: [PATCH 13/44] Ability to build multiplatform container image via maven --- pom.xml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 79d194d7..23129ed1 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,12 @@ 3.24.2 1.18.32 1.5.5.Final + + quay.io + debezium + platform-conductor + nightly + linux/amd64,linux/arm64/v8 @@ -64,7 +70,6 @@ io.debezium debezium-operator-api - 2.6.1.Final @@ -124,6 +129,10 @@ ${version.lombok} provided + + io.quarkus + quarkus-container-image-jib + io.quarkus quarkus-junit5 From 2b3a3f8750d74183c9bff822128814b42fe4ac80 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 22:58:06 +0200 Subject: [PATCH 14/44] Enabling db dev services only in dev profile --- src/main/resources/application.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 9584417d..6f9222ff 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -17,10 +17,9 @@ quarkus: name: type datasource: db-kind: postgresql - devservices: - enabled: true - port: 5432 - image-name: quay.io/debezium/postgres:14-alpine + hibernate-orm: + database: + generation: drop-and-create log: level: INFO @@ -34,6 +33,11 @@ quarkus: quarkus: debezium-outbox: remove-after-insert: false + datasource: + devservices: + enabled: true + port: 5432 + image-name: quay.io/debezium/postgres:14-alpine log: level: INFO category: From 1f433ba7628e335872bc3521c2159cdcd6038b11 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Sat, 11 May 2024 22:58:33 +0200 Subject: [PATCH 15/44] Added compose file --- compose.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 compose.yml diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..2b96f873 --- /dev/null +++ b/compose.yml @@ -0,0 +1,17 @@ +services: + conductor: + image: quay.io/debezium/platform-conductor:nightly + ports: + - "8080:8080" + environment: + QUARKUS_DATASOURCE_USERNAME: conductor + QUARKUS_DATASOURCE_PASSWORD: conductor + QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/conductor + postgres: + image: quay.io/debezium/postgres:14-alpine + ports: + - "5432:5432" + environment: + POSTGRES_USER: conductor + POSTGRES_PASSWORD: conductor + POSTGRES_DB: conductor \ No newline at end of file From 57af2422f4439f6ab3a09eb2f5bddaa9fb8c830d Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 13 May 2024 08:16:48 +0200 Subject: [PATCH 16/44] Always include swagger ui --- src/main/resources/application.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6f9222ff..b31e7c70 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -20,6 +20,8 @@ quarkus: hibernate-orm: database: generation: drop-and-create + swagger-ui: + always-include: true log: level: INFO From 65ce60fd65fffb6ae834790cbc63e4f555b91108 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 13 May 2024 08:40:16 +0200 Subject: [PATCH 17/44] Always pull conductor nightly image when running compose --- compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yml b/compose.yml index 2b96f873..b6542fce 100644 --- a/compose.yml +++ b/compose.yml @@ -1,6 +1,7 @@ services: conductor: image: quay.io/debezium/platform-conductor:nightly + pull_policy: always ports: - "8080:8080" environment: From 1f063112d05aab2349bd25dea602508defa78126 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 13 May 2024 11:51:49 +0200 Subject: [PATCH 18/44] Moved REST API compose under examples --- compose.yml => examples/compose-rest-api-only/compose.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename compose.yml => examples/compose-rest-api-only/compose.yml (100%) diff --git a/compose.yml b/examples/compose-rest-api-only/compose.yml similarity index 100% rename from compose.yml rename to examples/compose-rest-api-only/compose.yml From f4d465b48d7a6e085fc0aec20a3333c9c347deda Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 13 May 2024 11:52:52 +0200 Subject: [PATCH 19/44] Full conductor example using compose + kind --- .gitignore | 3 + examples/compose-kind-kafka/compose.yml | 36 +++++++++ examples/compose-kind-kafka/create-cluster.sh | 63 ++++++++++++++++ examples/compose-kind-kafka/destroy.sh | 4 + examples/compose-kind-kafka/env.sh | 14 ++++ .../k8s/database/001_postgresql.yml | 73 +++++++++++++++++++ .../k8s/kafka/001_kafka.yml | 33 +++++++++ .../payloads/destination.json | 11 +++ .../compose-kind-kafka/payloads/pipeline.json | 12 +++ .../compose-kind-kafka/payloads/source.json | 17 +++++ 10 files changed, 266 insertions(+) create mode 100644 examples/compose-kind-kafka/compose.yml create mode 100755 examples/compose-kind-kafka/create-cluster.sh create mode 100755 examples/compose-kind-kafka/destroy.sh create mode 100644 examples/compose-kind-kafka/env.sh create mode 100644 examples/compose-kind-kafka/k8s/database/001_postgresql.yml create mode 100644 examples/compose-kind-kafka/k8s/kafka/001_kafka.yml create mode 100644 examples/compose-kind-kafka/payloads/destination.json create mode 100644 examples/compose-kind-kafka/payloads/pipeline.json create mode 100644 examples/compose-kind-kafka/payloads/source.json diff --git a/.gitignore b/.gitignore index 8c7863e7..b52cc8f9 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ nb-configuration.xml # Plugin directory /.quarkus/cli/plugins/ + +# Generated Intenral Kind kubeconfig +examples/compose-kind-kafka/kubeconfig diff --git a/examples/compose-kind-kafka/compose.yml b/examples/compose-kind-kafka/compose.yml new file mode 100644 index 00000000..54b7519e --- /dev/null +++ b/examples/compose-kind-kafka/compose.yml @@ -0,0 +1,36 @@ +services: + conductor: + image: quay.io/debezium/platform-conductor:nightly + pull_policy: always + ports: + - "8080:8080" + environment: + QUARKUS_DATASOURCE_USERNAME: conductor + QUARKUS_DATASOURCE_PASSWORD: conductor + QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/conductor + CONDUCTOR_WATCHER_ENABLED: true + QUARKUS_KUBERNETES_CLIENT_NAMESPACE: debezium + volumes: + - type: bind + source: ./kubeconfig + target: /home/jboss/.kube/config + networks: + - conductor + - kind + postgres: + image: quay.io/debezium/postgres:14-alpine + ports: + - "5432:5432" + environment: + POSTGRES_USER: conductor + POSTGRES_PASSWORD: conductor + POSTGRES_DB: conductor + networks: + - conductor +volumes: + conductor: +networks: + conductor: + external: false + kind: + external: true diff --git a/examples/compose-kind-kafka/create-cluster.sh b/examples/compose-kind-kafka/create-cluster.sh new file mode 100755 index 00000000..38d44745 --- /dev/null +++ b/examples/compose-kind-kafka/create-cluster.sh @@ -0,0 +1,63 @@ +source env.sh + +echo "### DEBEZIUM DEMO ENVIRONMENT ###" +# Create cluster +echo ">>> Creating Cluster" +kind create cluster --name $CLUSTER +kind get kubeconfig --internal --name $CLUSTER > kubeconfig +kubectl cluster-info --context kind-$CLUSTER + +# Create namespace +echo ">>> Creating namespace" +kubectl create namespace $NAMESPACE +kubectl config set-context --current --namespace $NAMESPACE + +# Install operators +if $DBZ_INSTALL_OPERATOR; then + echo ">>> Installing Debezium operator" + echo ">> Add helm repo" + helm repo add debezium https://charts.debezium.io + echo ">> Deploy operator" + helm install debezium-operator debezium/debezium-operator --version $DBZ_OPERATOR_VERSION --namespace $NAMESPACE +fi + +if $STRIMZI_INSTALL_OPERATOR; then + echo ">>> Installing Strimzi operator" + echo ">> Add helm repo" + helm repo add strimzi https://strimzi.io/charts/ + echo ">> Deploy operator" + helm install strimzi-operator strimzi/strimzi-kafka-operator --version $STRIMZI_OPERATOR_VERSION --namespace $NAMESPACE +fi + +if $DBZ_INSTALL_OPERATOR; then + kubectl wait --for=condition=Available deployments/debezium-operator --timeout=$TIMEOUT -n $NAMESPACE + echo ">>> Debezium operator ready" +fi + +if $STRIMZI_INSTALL_OPERATOR; then + kubectl wait --for=condition=Available deployments/strimzi-cluster-operator --timeout=$TIMEOUT -n $NAMESPACE + echo ">>> Strimzi operator ready" +fi + +# Deploy Kafka and PostgreSQL +if $STRIMZI_DEPLOY_KAFKA; then + echo ">>> Deploy kafka" + kubectl create -f k8s/kafka/ -n $NAMESPACE +fi + +if $STRIMZI_DEPLOY_KAFKA; then + echo ">> Wait for kafka cluster" + kubectl wait --for=condition=Ready kafkas/dbz-kafka --timeout=$TIMEOUT -n $NAMESPACE + echo ">> Kafka cluster ready" +fi + +if $POSTGRES_DEPLOY; then + echo ">>> Deploy PostgreSQL" + kubectl create -f k8s/database/ -n $NAMESPACE +fi + +if $POSTGRES_DEPLOY; then + echo ">> Wait for PostgreSQL" + kubectl wait --for=condition=Available deployments/postgresql --timeout=$TIMEOUT -n $NAMESPACE + echo ">> PostgreSQL ready" +fi \ No newline at end of file diff --git a/examples/compose-kind-kafka/destroy.sh b/examples/compose-kind-kafka/destroy.sh new file mode 100755 index 00000000..f575aa1d --- /dev/null +++ b/examples/compose-kind-kafka/destroy.sh @@ -0,0 +1,4 @@ +source env.sh + +kind delete cluster --name $CLUSTER +rm -f kubeconfig \ No newline at end of file diff --git a/examples/compose-kind-kafka/env.sh b/examples/compose-kind-kafka/env.sh new file mode 100644 index 00000000..04c3512e --- /dev/null +++ b/examples/compose-kind-kafka/env.sh @@ -0,0 +1,14 @@ +CLUSTER=debezium +NAMESPACE=debezium +TIMEOUT=720s + +DBZ_INSTALL_OPERATOR=true +DBZ_OPERATOR_VERSION=2.6.1-final + +STRIMZI_INSTALL_OPERATOR=true +STRIMZI_OPERATOR_VERSION=0.40.0 +STRIMZI_DEPLOY_KAFKA=true + +POSTGRES_DEPLOY=true + + diff --git a/examples/compose-kind-kafka/k8s/database/001_postgresql.yml b/examples/compose-kind-kafka/k8s/database/001_postgresql.yml new file mode 100644 index 00000000..e44d88d2 --- /dev/null +++ b/examples/compose-kind-kafka/k8s/database/001_postgresql.yml @@ -0,0 +1,73 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-credentials +type: opaque +data: + POSTGRES_DB: ZGViZXppdW0= + POSTGRES_USER: ZGViZXppdW0= + POSTGRES_PASSWORD: ZGViZXppdW0= +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: postgresql + labels: + app: postgresql +spec: + replicas: 1 + selector: + matchLabels: + app: postgresql + deployment: postgresql + template: + metadata: + labels: + app: postgresql + deployment: postgresql + spec: + containers: + - resources: {} + name: postgresql + envFrom: + - secretRef: + name: postgresql-credentials + ports: + - containerPort: 5432 + protocol: TCP + imagePullPolicy: IfNotPresent + livenessProbe: + initialDelaySeconds: 30 + tcpSocket: + port: 5432 + timeoutSeconds: 1 + readinessProbe: + exec: + command: + - "/bin/sh" + - "-i" + - "-c" + - "PGPASSWORD=${POSTGRES_PASSWORD} /usr/bin/psql -w -U ${POSTGRES_USER} -d ${POSTGRES_DB} -c 'SELECT 1'" + initialDelaySeconds: 5 + timeoutSeconds: 1 + terminationMessagePolicy: File + terminationMessagePath: /dev/termination-log + image: quay.io/debezium/example-postgres:latest + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + strategy: + type: Recreate +--- +apiVersion: v1 +kind: Service +metadata: + name: postgresql +spec: + selector: + app: postgresql + deployment: postgresql + ports: + - name: db + port: 5432 + targetPort: 5432 diff --git a/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml b/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml new file mode 100644 index 00000000..25bc69bd --- /dev/null +++ b/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml @@ -0,0 +1,33 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: dbz-kafka +spec: + kafka: + version: 3.7.0 + replicas: 1 + listeners: + - name: plain + port: 9092 + type: internal + tls: false + - name: tls + port: 9093 + type: internal + tls: true + config: + offsets.topic.replication.factor: 1 + transaction.state.log.replication.factor: 1 + transaction.state.log.min.isr: 1 + default.replication.factor: 1 + min.insync.replicas: 1 + inter.broker.protocol.version: "3.7" + storage: + type: ephemeral + zookeeper: + replicas: 1 + storage: + type: ephemeral + entityOperator: + topicOperator: {} + userOperator: {} \ No newline at end of file diff --git a/examples/compose-kind-kafka/payloads/destination.json b/examples/compose-kind-kafka/payloads/destination.json new file mode 100644 index 00000000..7cdd0a1d --- /dev/null +++ b/examples/compose-kind-kafka/payloads/destination.json @@ -0,0 +1,11 @@ +{ + "name": "test-destination", + "type": "kafka", + "schema": "dummy", + "vaults": [], + "config": { + "producer.bootstrap.servers": "dbz-kafka-kafka-bootstrap.debezium:9092", + "producer.key.serializer": "org.apache.kafka.common.serialization.StringSerializer", + "producer.value.serializer": "org.apache.kafka.common.serialization.StringSerializer" + } +} \ No newline at end of file diff --git a/examples/compose-kind-kafka/payloads/pipeline.json b/examples/compose-kind-kafka/payloads/pipeline.json new file mode 100644 index 00000000..2652f28a --- /dev/null +++ b/examples/compose-kind-kafka/payloads/pipeline.json @@ -0,0 +1,12 @@ +{ + "name": "test-pipeline", + "source": { + "id": 1, + "name": "test-source" + }, + "destination": { + "id": 1, + "name": "test-destination" + }, + "logLevel": "debug" +} \ No newline at end of file diff --git a/examples/compose-kind-kafka/payloads/source.json b/examples/compose-kind-kafka/payloads/source.json new file mode 100644 index 00000000..a2a39905 --- /dev/null +++ b/examples/compose-kind-kafka/payloads/source.json @@ -0,0 +1,17 @@ +{ + "name": "test-source", + "type": "io.debezium.connector.postgresql.PostgresConnector", + "schema": "dummy", + "vaults": [], + "config": { + "offset.storage": "org.apache.kafka.connect.storage.MemoryOffsetBackingStore", + "database.history": "io.debezium.relational.history.MemorySchemaHistory", + "database.hostname": "postgresql", + "database.port": 5432, + "database.user": "debezium", + "database.password": "debezium", + "database.dbname": "debezium", + "topic.prefix": "inventory", + "schema.include.list": "inventory" + } +} \ No newline at end of file From 785a5f2cc1f9a4e404f8b25f0f8e551defd6950b Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 14 May 2024 11:04:24 +0200 Subject: [PATCH 20/44] Vaults now also include values --- .../java/io/debezium/platform/data/model/VaultEntity.java | 6 +++--- src/main/java/io/debezium/platform/domain/views/Vault.java | 6 +++--- src/test/java/io/debezium/platform/ServiceTest.java | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/debezium/platform/data/model/VaultEntity.java b/src/main/java/io/debezium/platform/data/model/VaultEntity.java index cceb3fda..808ca5ce 100644 --- a/src/main/java/io/debezium/platform/data/model/VaultEntity.java +++ b/src/main/java/io/debezium/platform/data/model/VaultEntity.java @@ -9,8 +9,8 @@ import org.hibernate.annotations.JdbcTypeCode; import org.hibernate.type.SqlTypes; -import java.util.LinkedList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; @Entity(name = "Vault") @Getter @@ -23,5 +23,5 @@ public class VaultEntity { private String name; private boolean plaintext = false; @JdbcTypeCode(SqlTypes.JSON) - private List keys = new LinkedList<>(); + private Map items = new HashMap<>(); } diff --git a/src/main/java/io/debezium/platform/domain/views/Vault.java b/src/main/java/io/debezium/platform/domain/views/Vault.java index c726375d..f6a677ed 100644 --- a/src/main/java/io/debezium/platform/domain/views/Vault.java +++ b/src/main/java/io/debezium/platform/domain/views/Vault.java @@ -7,7 +7,7 @@ import io.debezium.platform.data.model.VaultEntity; import io.debezium.platform.domain.views.refs.VaultReference; -import java.util.List; +import java.util.Map; @EntityView(VaultEntity.class) @CreatableEntityView @@ -15,9 +15,9 @@ public interface Vault extends VaultReference { boolean isPlaintext(); @MappingSingular - List getKeys(); + Map getItems(); void setName(String name); void setPlaintext(boolean plaintext); - void setKeys(List keys); + void setItems(Map items); } diff --git a/src/test/java/io/debezium/platform/ServiceTest.java b/src/test/java/io/debezium/platform/ServiceTest.java index 3870bfb5..3c086120 100644 --- a/src/test/java/io/debezium/platform/ServiceTest.java +++ b/src/test/java/io/debezium/platform/ServiceTest.java @@ -56,10 +56,10 @@ public class ServiceTest { public void createVault() { var vault1 = evm.create(Vault.class); vault1.setName("vault1"); - vault1.setKeys(List.of("foo", "bar")); + vault1.setItems(Map.of("foo", "bar", "baz", "qux")); var vault2 = evm.create(Vault.class); vault2.setName("vault2"); - vault2.setKeys(List.of("baz", "qux")); + vault2.setItems(Map.of("baz", "qux")); vaults.add(vaultService.create(vault1)); vaults.add(vaultService.create(vault2)); @@ -69,7 +69,7 @@ public void createVault() { @Order(10) public void updateVault() { var v = vaults.get(0); - v.setKeys(List.of("bar")); + v.setItems(Map.of("foo", "bar")); v = vaultService.update(v); vaults.set(0, v); } From 4bf301123003644dc4fdfe79b5677abe26fc8844 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 14 May 2024 11:07:50 +0200 Subject: [PATCH 21/44] Properly setting many-to-many relatinships in order to generate pkeys on join tables --- .../platform/data/model/DestinationEntity.java | 8 ++++---- .../debezium/platform/data/model/PipelineEntity.java | 5 ++++- .../io/debezium/platform/data/model/SourceEntity.java | 8 ++++---- .../debezium/platform/data/model/TransformEntity.java | 9 ++++++--- .../platform/domain/views/PipelineComponent.java | 6 +++--- src/test/java/io/debezium/platform/ServiceTest.java | 11 ++++++----- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/java/io/debezium/platform/data/model/DestinationEntity.java b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java index 9dc55c77..e4fe54a4 100644 --- a/src/main/java/io/debezium/platform/data/model/DestinationEntity.java +++ b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java @@ -12,9 +12,9 @@ import org.hibernate.annotations.JdbcTypeCode; import org.hibernate.type.SqlTypes; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; import java.util.Map; +import java.util.Set; @Entity(name = "Destination") @Getter @@ -30,8 +30,8 @@ public class DestinationEntity { @NotEmpty private String schema; @ManyToMany - @JoinTable(inverseJoinColumns = @JoinColumn(name = "vault_id")) - private List vaults = new LinkedList<>(); + @JoinTable(joinColumns = @JoinColumn(name = "destination_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) + private Set vaults = new HashSet<>(); @JdbcTypeCode(SqlTypes.JSON) private Map config; } diff --git a/src/main/java/io/debezium/platform/data/model/PipelineEntity.java b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java index 1c6595f5..875f3f75 100644 --- a/src/main/java/io/debezium/platform/data/model/PipelineEntity.java +++ b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java @@ -7,6 +7,7 @@ import jakarta.persistence.JoinTable; import jakarta.persistence.ManyToMany; import jakarta.persistence.ManyToOne; +import jakarta.persistence.OrderColumn; import jakarta.validation.constraints.NotEmpty; import lombok.Getter; import lombok.Setter; @@ -29,8 +30,10 @@ public class PipelineEntity { @ManyToOne private DestinationEntity destination; @ManyToMany - @JoinTable(inverseJoinColumns = @JoinColumn(name = "transform_id")) + @JoinTable(joinColumns = @JoinColumn(name = "pipeline_id"), inverseJoinColumns = @JoinColumn(name = "transform_id")) + @OrderColumn private List transforms = new LinkedList<>(); @NotEmpty private String logLevel = "info"; } + diff --git a/src/main/java/io/debezium/platform/data/model/SourceEntity.java b/src/main/java/io/debezium/platform/data/model/SourceEntity.java index d11b0d64..59335793 100644 --- a/src/main/java/io/debezium/platform/data/model/SourceEntity.java +++ b/src/main/java/io/debezium/platform/data/model/SourceEntity.java @@ -12,9 +12,9 @@ import org.hibernate.annotations.JdbcTypeCode; import org.hibernate.type.SqlTypes; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; import java.util.Map; +import java.util.Set; @Entity(name = "Source") @Getter @@ -30,8 +30,8 @@ public class SourceEntity { @NotEmpty public String schema; @ManyToMany - @JoinTable(inverseJoinColumns = @JoinColumn(name = "vault_id")) - public List vaults = new LinkedList<>(); + @JoinTable(joinColumns = @JoinColumn(name = "source_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) + public Set vaults = new HashSet<>(); @JdbcTypeCode(SqlTypes.JSON) public Map config; } diff --git a/src/main/java/io/debezium/platform/data/model/TransformEntity.java b/src/main/java/io/debezium/platform/data/model/TransformEntity.java index 82557349..5aa51edf 100644 --- a/src/main/java/io/debezium/platform/data/model/TransformEntity.java +++ b/src/main/java/io/debezium/platform/data/model/TransformEntity.java @@ -3,6 +3,8 @@ import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.JoinTable; import jakarta.persistence.ManyToMany; import jakarta.validation.constraints.NotEmpty; import lombok.Getter; @@ -10,9 +12,9 @@ import org.hibernate.annotations.JdbcTypeCode; import org.hibernate.type.SqlTypes; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; import java.util.Map; +import java.util.Set; @Entity(name = "Transform") @Getter @@ -28,7 +30,8 @@ public class TransformEntity { @NotEmpty private String schema; @ManyToMany - private List vaults = new LinkedList<>(); + @JoinTable(joinColumns = @JoinColumn(name = "transform_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) + private Set vaults = new HashSet<>(); @JdbcTypeCode(SqlTypes.JSON) private Map config; } diff --git a/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java index d58eda28..2db07b3a 100644 --- a/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java +++ b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java @@ -5,21 +5,21 @@ import io.debezium.platform.domain.views.refs.VaultReference; import jakarta.validation.constraints.NotEmpty; -import java.util.List; import java.util.Map; +import java.util.Set; public interface PipelineComponent extends NamedView { @NotEmpty String getType(); @NotEmpty String getSchema(); - List getVaults(); + Set getVaults(); @MappingSingular Map getConfig(); void setType(String type); void setName(String name); void setSchema(String schema); - void setVaults(List vaults); + void setVaults(Set vaults); void setConfig(Map config); } diff --git a/src/test/java/io/debezium/platform/ServiceTest.java b/src/test/java/io/debezium/platform/ServiceTest.java index 3c086120..af38da34 100644 --- a/src/test/java/io/debezium/platform/ServiceTest.java +++ b/src/test/java/io/debezium/platform/ServiceTest.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; @QuarkusTest @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -83,14 +84,14 @@ public void createSources() { source1.setName("source1"); source1.setSchema("schemaXY"); source1.setType("io.debezium.connector.MongoDbConnector"); - source1.setVaults(List.of(vaultRef(1))); + source1.setVaults(Set.of(vaultRef(1))); source1.setConfig(Map.of("mongodb.connection.string", "mongodb://localhost:27017")); var source2 = evm.create(Source.class); source2.setName("source2"); source2.setSchema("schemaXY"); source2.setType("io.debezium.connector.MongoDbConnector"); - source2.setVaults(List.of(vaultRef(0))); + source2.setVaults(Set.of(vaultRef(0))); source2.setConfig(Map.of("mongodb.connection.string", "mongodb://localhost:37017")); sources.add(sourceService.create(source1)); @@ -106,14 +107,14 @@ public void createDestinations() { destination1.setName("destination2"); destination1.setSchema("schemaDXY"); destination1.setType("pubsub"); - destination1.setVaults(List.of(vaultRef(0),vaultRef(1))); + destination1.setVaults(Set.of(vaultRef(0),vaultRef(1))); destination1.setConfig(Map.of("foo", "bar")); var destination2 = evm.create(Destination.class); destination2.setName("destination2"); destination2.setSchema("schemaDYZ"); destination2.setType("redis"); - destination2.setVaults(List.of(vaultRef(0))); + destination2.setVaults(Set.of(vaultRef(0))); destination2.setConfig(Map.of("bar", "baz")); destinations.add(destinationService.create(destination1)); @@ -129,7 +130,7 @@ public void createTransform() { transform1.setName("transform1"); transform1.setSchema("schemaASD"); transform1.setType("io.example.SomeTransform"); - transform1.setVaults(List.of(vaultRef(0),vaultRef(1))); + transform1.setVaults(Set.of(vaultRef(0),vaultRef(1))); transform1.setConfig(Map.of("baz", "qux")); transforms.add(transformService.create(transform1)); From 002adc436195fe53028e29944d35ae16451e34a1 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 14 May 2024 11:09:13 +0200 Subject: [PATCH 22/44] Workaround for BP delete issue --- .../platform/domain/AbstractService.java | 38 +++++++++++++++--- .../platform/domain/DestinationService.java | 4 +- .../platform/domain/PipelineService.java | 40 ++++--------------- .../platform/domain/SourceService.java | 4 +- .../platform/domain/TransformService.java | 4 +- .../platform/domain/VaultService.java | 38 +++--------------- .../domain/views/refs/PipelineReference.java | 10 +++++ .../io/debezium/platform/ServiceTest.java | 20 +++++++--- 8 files changed, 74 insertions(+), 84 deletions(-) create mode 100644 src/main/java/io/debezium/platform/domain/views/refs/PipelineReference.java diff --git a/src/main/java/io/debezium/platform/domain/AbstractService.java b/src/main/java/io/debezium/platform/domain/AbstractService.java index ba46ecaf..3e3ffe49 100644 --- a/src/main/java/io/debezium/platform/domain/AbstractService.java +++ b/src/main/java/io/debezium/platform/domain/AbstractService.java @@ -15,26 +15,36 @@ import static jakarta.transaction.Transactional.TxType.REQUIRED; import static jakarta.transaction.Transactional.TxType.SUPPORTS; +/** + * Service ancestor + * + * @param EntityType + * @param Updatable View Type + * @param Reference View Type + */ @Transactional(REQUIRED) -public class AbstractService { +public class AbstractService { EntityManager em; CriteriaBuilderFactory cbf; EntityViewManager evm; - Class viewType; Class entityType; + Class viewType; + Class referenceViewType; public AbstractService( Class entityType, Class viewType, + Class referenceViewType, EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + this.entityType = entityType; + this.viewType = viewType; + this.referenceViewType = referenceViewType; this.em = em; this.cbf = cbf; this.evm = evm; - this.entityType = entityType; - this.viewType = viewType; } public AbstractService() { @@ -45,6 +55,11 @@ protected CriteriaBuilder cb() { return cbf.create(em, entityType); } + @Transactional(SUPPORTS) + public V viewAs(T view, Class newViewType) { + return evm.convert(view, newViewType); + } + @Transactional(SUPPORTS) public List list() { return evm.applySetting(EntityViewSetting.create(viewType), cb()) @@ -64,15 +79,26 @@ public Optional findByIdAs(Class viewType, Long id) { public T create(@Valid T view) { evm.save(em, view); + onChange(view); return view; } public T update(@Valid T view) { evm.save(em, view); - return evm.find(em, viewType, view.getId()); + onChange(view); + return view; } public void delete(long id) { - evm.remove(em, viewType, id); + evm.remove(em, referenceViewType, id); + onChange(id); + } + + protected void onChange(T view) { + // default no-op + } + + protected void onChange(Long id) { + // default no-op } } diff --git a/src/main/java/io/debezium/platform/domain/DestinationService.java b/src/main/java/io/debezium/platform/domain/DestinationService.java index 2d900bee..d2f0133f 100644 --- a/src/main/java/io/debezium/platform/domain/DestinationService.java +++ b/src/main/java/io/debezium/platform/domain/DestinationService.java @@ -15,10 +15,10 @@ import static jakarta.transaction.Transactional.TxType.SUPPORTS; @ApplicationScoped -public class DestinationService extends AbstractService { +public class DestinationService extends AbstractService { public DestinationService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { - super(DestinationEntity.class, Destination.class, em, cbf, evm); + super(DestinationEntity.class, Destination.class, DestinationReference.class, em, cbf, evm); } @Transactional(SUPPORTS) diff --git a/src/main/java/io/debezium/platform/domain/PipelineService.java b/src/main/java/io/debezium/platform/domain/PipelineService.java index 4ad0fdfe..0408b30b 100644 --- a/src/main/java/io/debezium/platform/domain/PipelineService.java +++ b/src/main/java/io/debezium/platform/domain/PipelineService.java @@ -8,17 +8,15 @@ import io.debezium.platform.data.model.PipelineEntity; import io.debezium.platform.domain.views.Pipeline; import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.domain.views.refs.PipelineReference; import io.debezium.platform.environment.watcher.events.PipelineEvent; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Event; import jakarta.inject.Inject; import jakarta.persistence.EntityManager; -import jakarta.transaction.Transactional; - -import java.util.Optional; @ApplicationScoped -public class PipelineService extends AbstractService { +public class PipelineService extends AbstractService { @Inject Event> event; @@ -27,42 +25,18 @@ public class PipelineService extends AbstractService { ObjectMapper objectMapper; public PipelineService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { - super(PipelineEntity.class, Pipeline.class, em, cbf, evm); - } - - @Transactional(Transactional.TxType.SUPPORTS) - public Optional findByIdFlat(Long id) { - var result = evm.find(em, PipelineFlat.class, id); - return Optional.ofNullable(result); + super(PipelineEntity.class, Pipeline.class, PipelineReference.class, em, cbf, evm); } - @Override - public Pipeline create(Pipeline view) { - var result = super.create(view); - fireUpdateEvent(result.getId()); - return result; - } @Override - public Pipeline update(Pipeline view) { - var result = super.update(view); - fireUpdateEvent(result.getId()); - return result; + protected void onChange(Pipeline view) { + var flat =findByIdAs(PipelineFlat.class, view.getId()).orElseThrow(); + event.fire(PipelineEvent.update(flat, objectMapper)); } @Override - public void delete(long id) { - super.delete(id); - fireDeleteEvent(id); - } - - private void fireUpdateEvent(Long id) { - var flat = findByIdFlat(id); - flat.ifPresent(pipeline -> event.fire(PipelineEvent.update(pipeline, objectMapper))); - } - - private void fireDeleteEvent(Long id) { + protected void onChange(Long id) { event.fire(PipelineEvent.delete(id)); } - } diff --git a/src/main/java/io/debezium/platform/domain/SourceService.java b/src/main/java/io/debezium/platform/domain/SourceService.java index c13d35ee..4b3ee4fe 100644 --- a/src/main/java/io/debezium/platform/domain/SourceService.java +++ b/src/main/java/io/debezium/platform/domain/SourceService.java @@ -20,10 +20,10 @@ import static jakarta.transaction.Transactional.TxType.SUPPORTS; @ApplicationScoped -public class SourceService extends AbstractService { +public class SourceService extends AbstractService { public SourceService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { - super(SourceEntity.class, Source.class, em, cbf, evm); + super(SourceEntity.class, Source.class, SourceReference.class, em, cbf, evm); } @Transactional(SUPPORTS) diff --git a/src/main/java/io/debezium/platform/domain/TransformService.java b/src/main/java/io/debezium/platform/domain/TransformService.java index 77d3b03c..ec143229 100644 --- a/src/main/java/io/debezium/platform/domain/TransformService.java +++ b/src/main/java/io/debezium/platform/domain/TransformService.java @@ -17,10 +17,10 @@ import static jakarta.transaction.Transactional.TxType.SUPPORTS; @ApplicationScoped -public class TransformService extends AbstractService { +public class TransformService extends AbstractService { public TransformService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { - super(TransformEntity.class, Transform.class, em, cbf, evm); + super(TransformEntity.class, Transform.class, TransformReference.class, em, cbf, evm); } @Transactional(SUPPORTS) diff --git a/src/main/java/io/debezium/platform/domain/VaultService.java b/src/main/java/io/debezium/platform/domain/VaultService.java index ea668625..e79f7ed9 100644 --- a/src/main/java/io/debezium/platform/domain/VaultService.java +++ b/src/main/java/io/debezium/platform/domain/VaultService.java @@ -13,15 +13,12 @@ import jakarta.enterprise.event.Event; import jakarta.inject.Inject; import jakarta.persistence.EntityManager; -import jakarta.transaction.Transactional; - -import java.util.Optional; import static jakarta.transaction.Transactional.TxType.SUPPORTS; @ApplicationScoped -public class VaultService extends AbstractService { +public class VaultService extends AbstractService { @Inject Event> event; @@ -30,41 +27,16 @@ public class VaultService extends AbstractService { ObjectMapper objectMapper; public VaultService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { - super(VaultEntity.class, Vault.class, em, cbf, evm); - } - - @Transactional(SUPPORTS) - public Optional findReferenceById(Long id) { - var result = evm.find(em, VaultReference.class, id); - return Optional.ofNullable(result); - } - - @Override - public Vault create(Vault view) { - var result = super.create(view); - fireUpdateEvent(result.getId()); - return result; + super(VaultEntity.class, Vault.class, VaultReference.class, em, cbf, evm); } @Override - public Vault update(Vault view) { - var result = super.update(view); - fireUpdateEvent(result.getId()); - return result; + protected void onChange(Vault view) { + event.fire(VaultEvent.update(view, objectMapper)); } @Override - public void delete(long id) { - super.delete(id); - fireDeleteEvent(id); - } - - private void fireUpdateEvent(Long id) { - var flat = findById(id); - flat.ifPresent(vault -> event.fire(VaultEvent.update(vault, objectMapper))); - } - - private void fireDeleteEvent(Long id) { + protected void onChange(Long id) { event.fire(VaultEvent.delete(id)); } } diff --git a/src/main/java/io/debezium/platform/domain/views/refs/PipelineReference.java b/src/main/java/io/debezium/platform/domain/views/refs/PipelineReference.java new file mode 100644 index 00000000..93a7654c --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/views/refs/PipelineReference.java @@ -0,0 +1,10 @@ +package io.debezium.platform.domain.views.refs; + +import com.blazebit.persistence.view.EntityView; +import io.debezium.platform.data.model.DestinationEntity; +import io.debezium.platform.data.model.PipelineEntity; +import io.debezium.platform.domain.views.base.NamedView; + +@EntityView(PipelineEntity.class) +public interface PipelineReference extends NamedView { +} diff --git a/src/test/java/io/debezium/platform/ServiceTest.java b/src/test/java/io/debezium/platform/ServiceTest.java index af38da34..04e42311 100644 --- a/src/test/java/io/debezium/platform/ServiceTest.java +++ b/src/test/java/io/debezium/platform/ServiceTest.java @@ -180,6 +180,17 @@ public void listPipelines() { Assertions.assertThat(list).hasSize(1); } + @Test + @Order(61) + public void deletePipeline() { + pipelineService.delete(pipeline.getId()); + + var found = pipelineService.findById(pipeline.getId()); + Assertions.assertThat(found).isEmpty(); + Assertions.assertThat(destinationService.list()).hasSize(2); + Assertions.assertThat(sourceService.list()).hasSize(2); + } + @Test @Order(100) public void foo() { @@ -187,17 +198,14 @@ public void foo() { } public VaultReference vaultRef(int idx) { - var vault = vaults.get(idx); - return vaultService.findReferenceById(vault.getId()).orElseThrow(); + return vaultService.viewAs(vaults.get(idx), VaultReference.class); } public SourceReference sourceRef(int idx) { - var source = sources.get(idx); - return sourceService.findReferenceById(source.getId()).orElseThrow(); + return sourceService.viewAs(sources.get(idx), SourceReference.class); } public DestinationReference destinationRef(int idx) { - var destination = destinations.get(idx); - return destinationService.findReferenceById(destination.getId()).orElseThrow(); + return destinationService.viewAs(destinations.get(idx), DestinationReference.class); } } From afaa4853816fd8f1ddd5906e7ce0df3d4283dbdd Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 14 May 2024 11:09:39 +0200 Subject: [PATCH 23/44] Fixed typo in DS configuration --- .../environment/operator/OperatorPipelineController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java index fca5f05c..20e953b4 100644 --- a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java @@ -31,7 +31,7 @@ public OperatorPipelineController(KubernetesClient k8s) { public void deploy(PipelineFlat pipeline) { // Create DS quarkus configuration var quarkusConfig = new ConfigProperties(); - quarkusConfig.setProps("log.leve", pipeline.getLogLevel()); + quarkusConfig.setProps("log.level", pipeline.getLogLevel()); var dsQuarkus = new QuarkusBuilder() .withConfig(quarkusConfig) .build(); From d0cefcde084ddcb6feb6d02a1d8fdeff1fdbbe2e Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Wed, 22 May 2024 10:32:37 +0200 Subject: [PATCH 24/44] Added unique constrain on names and description field --- examples/compose-kind-kafka/payloads/destination.json | 1 + examples/compose-kind-kafka/payloads/pipeline.json | 1 + examples/compose-kind-kafka/payloads/source.json | 1 + .../io/debezium/platform/data/model/DestinationEntity.java | 5 +++++ .../java/io/debezium/platform/data/model/PipelineEntity.java | 3 +++ .../java/io/debezium/platform/data/model/SourceEntity.java | 5 +++++ .../io/debezium/platform/data/model/TransformEntity.java | 5 +++++ .../java/io/debezium/platform/data/model/VaultEntity.java | 3 +++ .../io/debezium/platform/domain/views/PipelineComponent.java | 2 ++ .../io/debezium/platform/domain/views/flat/PipelineFlat.java | 1 + 10 files changed, 27 insertions(+) diff --git a/examples/compose-kind-kafka/payloads/destination.json b/examples/compose-kind-kafka/payloads/destination.json index 7cdd0a1d..475cd1bc 100644 --- a/examples/compose-kind-kafka/payloads/destination.json +++ b/examples/compose-kind-kafka/payloads/destination.json @@ -1,6 +1,7 @@ { "name": "test-destination", "type": "kafka", + "description": "Some funny destination", "schema": "dummy", "vaults": [], "config": { diff --git a/examples/compose-kind-kafka/payloads/pipeline.json b/examples/compose-kind-kafka/payloads/pipeline.json index 2652f28a..2e30ebfe 100644 --- a/examples/compose-kind-kafka/payloads/pipeline.json +++ b/examples/compose-kind-kafka/payloads/pipeline.json @@ -1,5 +1,6 @@ { "name": "test-pipeline", + "description": "It goes from here to there!", "source": { "id": 1, "name": "test-source" diff --git a/examples/compose-kind-kafka/payloads/source.json b/examples/compose-kind-kafka/payloads/source.json index a2a39905..180d99fe 100644 --- a/examples/compose-kind-kafka/payloads/source.json +++ b/examples/compose-kind-kafka/payloads/source.json @@ -1,5 +1,6 @@ { "name": "test-source", + "description": "Yummy data source", "type": "io.debezium.connector.postgresql.PostgresConnector", "schema": "dummy", "vaults": [], diff --git a/src/main/java/io/debezium/platform/data/model/DestinationEntity.java b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java index e4fe54a4..e44b0b54 100644 --- a/src/main/java/io/debezium/platform/data/model/DestinationEntity.java +++ b/src/main/java/io/debezium/platform/data/model/DestinationEntity.java @@ -1,5 +1,6 @@ package io.debezium.platform.data.model; +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; @@ -24,10 +25,14 @@ public class DestinationEntity { @GeneratedValue private Long id; @NotEmpty + @Column(unique = true, nullable = false) private String name; + private String description; @NotEmpty + @Column(nullable = false) private String type; @NotEmpty + @Column(nullable = false) private String schema; @ManyToMany @JoinTable(joinColumns = @JoinColumn(name = "destination_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) diff --git a/src/main/java/io/debezium/platform/data/model/PipelineEntity.java b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java index 875f3f75..2ae22721 100644 --- a/src/main/java/io/debezium/platform/data/model/PipelineEntity.java +++ b/src/main/java/io/debezium/platform/data/model/PipelineEntity.java @@ -1,5 +1,6 @@ package io.debezium.platform.data.model; +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; @@ -24,7 +25,9 @@ public class PipelineEntity { @GeneratedValue private Long id; @NotEmpty + @Column(unique = true, nullable = false) private String name; + private String description; @ManyToOne private SourceEntity source; @ManyToOne diff --git a/src/main/java/io/debezium/platform/data/model/SourceEntity.java b/src/main/java/io/debezium/platform/data/model/SourceEntity.java index 59335793..812b20ed 100644 --- a/src/main/java/io/debezium/platform/data/model/SourceEntity.java +++ b/src/main/java/io/debezium/platform/data/model/SourceEntity.java @@ -1,5 +1,6 @@ package io.debezium.platform.data.model; +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; @@ -24,10 +25,14 @@ public class SourceEntity { @GeneratedValue private Long id; @NotEmpty + @Column(unique = true, nullable = false) public String name; + private String description; @NotEmpty + @Column(nullable = false) public String type; @NotEmpty + @Column(nullable = false) public String schema; @ManyToMany @JoinTable(joinColumns = @JoinColumn(name = "source_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) diff --git a/src/main/java/io/debezium/platform/data/model/TransformEntity.java b/src/main/java/io/debezium/platform/data/model/TransformEntity.java index 5aa51edf..36bc7fab 100644 --- a/src/main/java/io/debezium/platform/data/model/TransformEntity.java +++ b/src/main/java/io/debezium/platform/data/model/TransformEntity.java @@ -1,5 +1,6 @@ package io.debezium.platform.data.model; +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; @@ -24,10 +25,14 @@ public class TransformEntity { @GeneratedValue private Long id; @NotEmpty + @Column(unique = true, nullable = false) private String name; + private String description; @NotEmpty + @Column(nullable = false) private String type; @NotEmpty + @Column(nullable = false) private String schema; @ManyToMany @JoinTable(joinColumns = @JoinColumn(name = "transform_id"), inverseJoinColumns = @JoinColumn(name = "vault_id")) diff --git a/src/main/java/io/debezium/platform/data/model/VaultEntity.java b/src/main/java/io/debezium/platform/data/model/VaultEntity.java index 808ca5ce..a02a24b3 100644 --- a/src/main/java/io/debezium/platform/data/model/VaultEntity.java +++ b/src/main/java/io/debezium/platform/data/model/VaultEntity.java @@ -1,5 +1,6 @@ package io.debezium.platform.data.model; +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; @@ -20,7 +21,9 @@ public class VaultEntity { @GeneratedValue private Long id; @NotEmpty + @Column(unique = true, nullable = false) private String name; + private String description; private boolean plaintext = false; @JdbcTypeCode(SqlTypes.JSON) private Map items = new HashMap<>(); diff --git a/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java index 2db07b3a..27ad98fe 100644 --- a/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java +++ b/src/main/java/io/debezium/platform/domain/views/PipelineComponent.java @@ -9,6 +9,7 @@ import java.util.Set; public interface PipelineComponent extends NamedView { + String getDescription(); @NotEmpty String getType(); @NotEmpty @@ -17,6 +18,7 @@ public interface PipelineComponent extends NamedView { @MappingSingular Map getConfig(); + void setDescription(String description); void setType(String type); void setName(String name); void setSchema(String schema); diff --git a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java index e194fa7a..6074a0b5 100644 --- a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java +++ b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java @@ -15,6 +15,7 @@ @EntityView(PipelineEntity.class) @UpdatableEntityView public interface PipelineFlat extends NamedView { + String getDescription(); @NotNull Source getSource(); @NotNull From fef40d0425f9f4b67dbbd67098a11994ba123f50 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Wed, 22 May 2024 14:19:51 +0200 Subject: [PATCH 25/44] Added the description filed to pipeline views --- src/main/java/io/debezium/platform/domain/views/Pipeline.java | 2 ++ .../io/debezium/platform/domain/views/flat/PipelineFlat.java | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/java/io/debezium/platform/domain/views/Pipeline.java b/src/main/java/io/debezium/platform/domain/views/Pipeline.java index 7d1a52aa..da375e58 100644 --- a/src/main/java/io/debezium/platform/domain/views/Pipeline.java +++ b/src/main/java/io/debezium/platform/domain/views/Pipeline.java @@ -17,6 +17,7 @@ @CreatableEntityView @UpdatableEntityView public interface Pipeline extends NamedView { + String getDescription(); @NotNull SourceReference getSource(); @NotNull @@ -25,6 +26,7 @@ public interface Pipeline extends NamedView { @NotEmpty String getLogLevel(); + void setDescription(String description); void setName(String name); void setSource(SourceReference source); void setDestination(DestinationReference destination); diff --git a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java index 6074a0b5..85146bb0 100644 --- a/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java +++ b/src/main/java/io/debezium/platform/domain/views/flat/PipelineFlat.java @@ -24,6 +24,7 @@ public interface PipelineFlat extends NamedView { @NotEmpty String getLogLevel(); + void setDescription(String description); void setName(String name); void setSource(Source source); void setDestination(Destination destination); From a983a56d3b89dd0c3bbc349438a8375330870e00 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Wed, 22 May 2024 16:51:04 +0200 Subject: [PATCH 26/44] Temporarily hardcoding some pipeline configuration --- .../environment/operator/OperatorPipelineController.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java index 20e953b4..935e7b05 100644 --- a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java @@ -31,7 +31,10 @@ public OperatorPipelineController(KubernetesClient k8s) { public void deploy(PipelineFlat pipeline) { // Create DS quarkus configuration var quarkusConfig = new ConfigProperties(); - quarkusConfig.setProps("log.level", pipeline.getLogLevel()); + quarkusConfig.setAllProps(Map.of( + "log.level", pipeline.getLogLevel(), + "log.console.json", false + )); var dsQuarkus = new QuarkusBuilder() .withConfig(quarkusConfig) .build(); @@ -40,6 +43,8 @@ public void deploy(PipelineFlat pipeline) { var source = pipeline.getSource(); var sourceConfig = new ConfigProperties(); sourceConfig.setAllProps(source.getConfig()); + sourceConfig.setProps("offset.storage", "org.apache.kafka.connect.storage.MemoryOffsetBackingStore"); + sourceConfig.setProps("database.history", "io.debezium.relational.history.MemorySchemaHistory"); var dsSource = new SourceBuilder() .withSourceClass(source.getType()) From 9482e88eec02b6012aa200d07e51ecbf7cd0f9fe Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 17 Jun 2024 13:21:26 +0200 Subject: [PATCH 27/44] Aligning with Debezium Operator 2.7.0.Beta2 --- examples/compose-kind-kafka/env.sh | 2 +- .../compose-kind-kafka/payloads/source.json | 2 -- pom.xml | 2 +- .../operator/OperatorPipelineController.java | 30 +++++++++++++++++-- src/main/resources/application.yml | 2 +- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/examples/compose-kind-kafka/env.sh b/examples/compose-kind-kafka/env.sh index 04c3512e..110b9af2 100644 --- a/examples/compose-kind-kafka/env.sh +++ b/examples/compose-kind-kafka/env.sh @@ -3,7 +3,7 @@ NAMESPACE=debezium TIMEOUT=720s DBZ_INSTALL_OPERATOR=true -DBZ_OPERATOR_VERSION=2.6.1-final +DBZ_OPERATOR_VERSION=2.7.0-beta2 STRIMZI_INSTALL_OPERATOR=true STRIMZI_OPERATOR_VERSION=0.40.0 diff --git a/examples/compose-kind-kafka/payloads/source.json b/examples/compose-kind-kafka/payloads/source.json index 180d99fe..aaae4602 100644 --- a/examples/compose-kind-kafka/payloads/source.json +++ b/examples/compose-kind-kafka/payloads/source.json @@ -5,8 +5,6 @@ "schema": "dummy", "vaults": [], "config": { - "offset.storage": "org.apache.kafka.connect.storage.MemoryOffsetBackingStore", - "database.history": "io.debezium.relational.history.MemorySchemaHistory", "database.hostname": "postgresql", "database.port": 5432, "database.user": "debezium", diff --git a/pom.xml b/pom.xml index 23129ed1..6e2cdeb7 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ io.debezium debezium-operator-api - 2.6.1.Final + 2.7.0.Beta2 io.quarkus diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java index 935e7b05..1299860d 100644 --- a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java @@ -6,7 +6,14 @@ import io.debezium.operator.api.model.DebeziumServerSpecBuilder; import io.debezium.operator.api.model.QuarkusBuilder; import io.debezium.operator.api.model.SinkBuilder; -import io.debezium.operator.api.model.SourceBuilder; +import io.debezium.operator.api.model.runtime.RuntimeBuilder; +import io.debezium.operator.api.model.runtime.metrics.JmxExporterBuilder; +import io.debezium.operator.api.model.runtime.metrics.MetricsBuilder; +import io.debezium.operator.api.model.source.OffsetBuilder; +import io.debezium.operator.api.model.source.SchemaHistoryBuilder; +import io.debezium.operator.api.model.source.SourceBuilder; +import io.debezium.operator.api.model.source.storage.offset.InMemoryOffsetStore; +import io.debezium.operator.api.model.source.storage.schema.InMemorySchemaHistoryStore; import io.debezium.platform.domain.views.flat.PipelineFlat; import io.debezium.platform.environment.PipelineController; import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; @@ -39,15 +46,31 @@ public void deploy(PipelineFlat pipeline) { .withConfig(quarkusConfig) .build(); + var dsRuntime = new RuntimeBuilder() + .withMetrics(new MetricsBuilder() + .withJmxExporter(new JmxExporterBuilder() + .withEnabled() + .build()) + .build()) + .build(); + // Create DS source configuration var source = pipeline.getSource(); var sourceConfig = new ConfigProperties(); sourceConfig.setAllProps(source.getConfig()); - sourceConfig.setProps("offset.storage", "org.apache.kafka.connect.storage.MemoryOffsetBackingStore"); - sourceConfig.setProps("database.history", "io.debezium.relational.history.MemorySchemaHistory"); + + // TODO: offset and schema history type should be configurable in the future + var offset = new OffsetBuilder() + .withMemory(new InMemoryOffsetStore()) + .build(); + var schemaHistory = new SchemaHistoryBuilder() + .withMemory(new InMemorySchemaHistoryStore()) + .build(); var dsSource = new SourceBuilder() .withSourceClass(source.getType()) + .withOffset(offset) + .withSchemaHistory(schemaHistory) .withConfig(sourceConfig) .build(); @@ -69,6 +92,7 @@ public void deploy(PipelineFlat pipeline) { .build()) .withSpec(new DebeziumServerSpecBuilder() .withQuarkus(dsQuarkus) + .withRuntime(dsRuntime) .withSource(dsSource) .withSink(dsSink) .build()) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b31e7c70..2a5c6c52 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -39,7 +39,7 @@ quarkus: devservices: enabled: true port: 5432 - image-name: quay.io/debezium/postgres:14-alpine + image-name: quay.io/debezium/postgres:16-alpine log: level: INFO category: From a876956b1f994a60b4813b51babd032fb45a6015 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 24 Jun 2024 14:18:18 +0200 Subject: [PATCH 28/44] WebSocket endpoint for streamining pipeline logs --- README.md | 55 ++-------- examples/compose-kind-kafka/compose.yml | 9 ++ examples/compose-kind-kafka/env.sh | 2 +- examples/compose-rest-api-only/compose.yml | 9 +- pom.xml | 10 +- .../platform/api/DestinationResource.java | 2 - .../platform/api/PipelineLogWebSocket.java | 82 ++++++++++++++ .../debezium/platform/api/SourceResource.java | 2 - .../platform/api/TransformResource.java | 2 - .../platform/domain/AbstractService.java | 6 +- .../platform/domain/LogStreamingService.java | 103 ++++++++++++++++++ .../platform/domain/PipelineService.java | 61 +++++++++-- .../platform/domain/VaultService.java | 9 +- .../environment/PipelineController.java | 44 ++++++++ .../platform/environment/logs/LogReader.java | 34 ++++++ .../operator/DevClusterInitializer.java | 74 +++++++++++++ .../environment/operator/DevCrdInstaller.java | 46 -------- .../OperatorEnvironmentController.java | 3 + .../operator/OperatorPipelineController.java | 45 ++++++++ .../operator/logs/KubernetesLogReader.java | 55 ++++++++++ .../platform/error/NotFoundException.java | 15 +++ src/main/resources/application.yml | 5 + 22 files changed, 552 insertions(+), 121 deletions(-) create mode 100644 src/main/java/io/debezium/platform/api/PipelineLogWebSocket.java create mode 100644 src/main/java/io/debezium/platform/domain/LogStreamingService.java create mode 100644 src/main/java/io/debezium/platform/environment/logs/LogReader.java create mode 100644 src/main/java/io/debezium/platform/environment/operator/DevClusterInitializer.java delete mode 100644 src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java create mode 100644 src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java create mode 100644 src/main/java/io/debezium/platform/error/NotFoundException.java diff --git a/README.md b/README.md index 64fa57b8..bed6bf75 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,11 @@ # debezium-platform-conductor -This project uses Quarkus, the Supersonic Subatomic Java Framework. +The back-end component of Debezium management platform. Conductor provides a set APIs which can be +used to orchestrate and control Debezium deployments. It's intended to be interacted with through a front-end client. -If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ . -## Running the application in dev mode - -You can run your application in dev mode that enables live coding using: -```shell script -./mvnw compile quarkus:dev -``` - -> **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/. - -## Packaging and running the application - -The application can be packaged using: -```shell script -./mvnw package -``` -It produces the `quarkus-run.jar` file in the `target/quarkus-app/` directory. -Be aware that it’s not an _über-jar_ as the dependencies are copied into the `target/quarkus-app/lib/` directory. - -The application is now runnable using `java -jar target/quarkus-app/quarkus-run.jar`. - -If you want to build an _über-jar_, execute the following command: -```shell script -./mvnw package -Dquarkus.package.type=uber-jar -``` - -The application, packaged as an _über-jar_, is now runnable using `java -jar target/*-runner.jar`. - -## Creating a native executable - -You can create a native executable using: -```shell script -./mvnw package -Dnative -``` - -Or, if you don't have GraalVM installed, you can run the native executable build in a container using: -```shell script -./mvnw package -Dnative -Dquarkus.native.container-build=true -``` - -You can then execute your native executable with: `./target/debezium-platform-conductor-0.1.0-SNAPSHOT-runner` - -If you want to learn more about building native executables, please consult https://quarkus.io/guides/maven-tooling. - -## Related Guides - -- YAML Configuration ([guide](https://quarkus.io/guides/config-yaml)): Use YAML to configure your Quarkus application +## Debezium Management Platform +Debezium Management Platform (Debezium Orchestra) aims to provide means to simplify the deployment of +Debezium to various environments in highly opinionated manner. The goal is not to provide +total control over environment specific configuration. To achieve this goal the platform uses +a data-centric view on Debezium components. \ No newline at end of file diff --git a/examples/compose-kind-kafka/compose.yml b/examples/compose-kind-kafka/compose.yml index 54b7519e..854c4d85 100644 --- a/examples/compose-kind-kafka/compose.yml +++ b/examples/compose-kind-kafka/compose.yml @@ -10,6 +10,7 @@ services: QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/conductor CONDUCTOR_WATCHER_ENABLED: true QUARKUS_KUBERNETES_CLIENT_NAMESPACE: debezium + QUARKUS_HTTP_CORS_ORIGINS: http://localhost:3000 volumes: - type: bind source: ./kubeconfig @@ -27,6 +28,14 @@ services: POSTGRES_DB: conductor networks: - conductor + stage: + container_name: stage + image: quay.io/debezium/platform-stage:latest + pull_policy: always + ports: + - "3000:3000" + networks: + - conductor volumes: conductor: networks: diff --git a/examples/compose-kind-kafka/env.sh b/examples/compose-kind-kafka/env.sh index 110b9af2..1667a13b 100644 --- a/examples/compose-kind-kafka/env.sh +++ b/examples/compose-kind-kafka/env.sh @@ -2,7 +2,7 @@ CLUSTER=debezium NAMESPACE=debezium TIMEOUT=720s -DBZ_INSTALL_OPERATOR=true +DBZ_INSTALL_OPERATOR=false DBZ_OPERATOR_VERSION=2.7.0-beta2 STRIMZI_INSTALL_OPERATOR=true diff --git a/examples/compose-rest-api-only/compose.yml b/examples/compose-rest-api-only/compose.yml index b6542fce..98cbbef0 100644 --- a/examples/compose-rest-api-only/compose.yml +++ b/examples/compose-rest-api-only/compose.yml @@ -8,6 +8,7 @@ services: QUARKUS_DATASOURCE_USERNAME: conductor QUARKUS_DATASOURCE_PASSWORD: conductor QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/conductor + QUARKUS_HTTP_CORS_ORIGINS: http://localhost:3000 postgres: image: quay.io/debezium/postgres:14-alpine ports: @@ -15,4 +16,10 @@ services: environment: POSTGRES_USER: conductor POSTGRES_PASSWORD: conductor - POSTGRES_DB: conductor \ No newline at end of file + POSTGRES_DB: conductor + stage: + container_name: stage + image: quay.io/debezium/platform-stage:latest + pull_policy: always + ports: + - "3000:3000" \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6e2cdeb7..8e9c026f 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ UTF-8 quarkus-bom io.quarkus.platform - 3.8.4 + 3.11.2 true 3.2.5 3.24.2 @@ -29,7 +29,7 @@ ${quarkus.platform.group-id} - ${quarkus.platform.artifact-id} + quarkus-bom ${quarkus.platform.version} pom import @@ -70,7 +70,7 @@ io.debezium debezium-operator-api - 2.7.0.Beta2 + 2.7.0-SNAPSHOT io.quarkus @@ -104,6 +104,10 @@ io.quarkus quarkus-smallrye-openapi + + io.quarkus + quarkus-websockets-next + com.blazebit blaze-persistence-integration-quarkus-3 diff --git a/src/main/java/io/debezium/platform/api/DestinationResource.java b/src/main/java/io/debezium/platform/api/DestinationResource.java index 6ff31b1d..01f8b6a2 100644 --- a/src/main/java/io/debezium/platform/api/DestinationResource.java +++ b/src/main/java/io/debezium/platform/api/DestinationResource.java @@ -11,7 +11,6 @@ import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriInfo; @@ -27,7 +26,6 @@ import org.jboss.logging.Logger; import java.net.URI; -import java.util.Objects; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; diff --git a/src/main/java/io/debezium/platform/api/PipelineLogWebSocket.java b/src/main/java/io/debezium/platform/api/PipelineLogWebSocket.java new file mode 100644 index 00000000..3f3e43e1 --- /dev/null +++ b/src/main/java/io/debezium/platform/api/PipelineLogWebSocket.java @@ -0,0 +1,82 @@ +package io.debezium.platform.api; + + +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.LogStreamingService; +import io.debezium.platform.error.NotFoundException; +import io.quarkus.websockets.next.InboundProcessingMode; +import io.quarkus.websockets.next.OnClose; +import io.quarkus.websockets.next.OnError; +import io.quarkus.websockets.next.OnOpen; +import io.quarkus.websockets.next.PathParam; +import io.quarkus.websockets.next.WebSocket; +import io.quarkus.websockets.next.WebSocketConnection; +import io.smallrye.common.annotation.RunOnVirtualThread; +import jakarta.inject.Inject; +import org.jboss.logging.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@WebSocket( + path = "/api/pipelines/{id}/logs/stream", + inboundProcessingMode = InboundProcessingMode.CONCURRENT +) +public class PipelineLogWebSocket { + + private static final org.slf4j.Logger log = LoggerFactory.getLogger(PipelineLogWebSocket.class); + @Inject + Logger logger; + + @Inject + PipelineService pipelineService; + + @Inject + LogStreamingService logStreamer; + + private final Map streamingTasks = new ConcurrentHashMap<>(); + + + @OnOpen + @RunOnVirtualThread + public void onOpen(@PathParam("id") String idString, WebSocketConnection connection) { + logger.infof("Connection '%s' requesting logs for pipeline '%s',", connection.id(), idString); + var id = Long.parseLong(idString); + + pipelineService + .streamLogs(id, connection::sendTextAndAwait) + .ifPresentOrElse( + task -> streamingTasks.put(connection.id(), task), + () -> { + throw new NotFoundException(id); + }); + } + + @OnError + public void onError(WebSocketConnection connection, @PathParam("id") String idString, NumberFormatException e) { + logger.warnf("Invalid pipeline id: %s", idString); + + connection.sendTextAndAwait("Invalid pipeline id"); + connection.closeAndAwait(); + } + + @OnError + public void onError(WebSocketConnection connection, NotFoundException e) { + logger.warnf("Pipeline not found: %s", e.getId()); + + connection.sendTextAndAwait("Pipeline not found"); + connection.closeAndAwait(); + } + + @OnClose + public void onClose(WebSocketConnection connection) { + logger.debugf("Connection: %s closed", connection.id()); + + var task = streamingTasks.remove(connection.id()); + if (task != null) { + task.stop(); + } + } + +} diff --git a/src/main/java/io/debezium/platform/api/SourceResource.java b/src/main/java/io/debezium/platform/api/SourceResource.java index 25aa42c0..4ca1fd61 100644 --- a/src/main/java/io/debezium/platform/api/SourceResource.java +++ b/src/main/java/io/debezium/platform/api/SourceResource.java @@ -11,7 +11,6 @@ import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriInfo; @@ -27,7 +26,6 @@ import org.jboss.logging.Logger; import java.net.URI; -import java.util.Objects; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; diff --git a/src/main/java/io/debezium/platform/api/TransformResource.java b/src/main/java/io/debezium/platform/api/TransformResource.java index c3fa4427..11f9b407 100644 --- a/src/main/java/io/debezium/platform/api/TransformResource.java +++ b/src/main/java/io/debezium/platform/api/TransformResource.java @@ -11,7 +11,6 @@ import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriInfo; @@ -27,7 +26,6 @@ import org.jboss.logging.Logger; import java.net.URI; -import java.util.Objects; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; diff --git a/src/main/java/io/debezium/platform/domain/AbstractService.java b/src/main/java/io/debezium/platform/domain/AbstractService.java index 3e3ffe49..a86be238 100644 --- a/src/main/java/io/debezium/platform/domain/AbstractService.java +++ b/src/main/java/io/debezium/platform/domain/AbstractService.java @@ -94,11 +94,13 @@ public void delete(long id) { onChange(id); } - protected void onChange(T view) { + @Transactional(REQUIRED) + public void onChange(T view) { // default no-op } - protected void onChange(Long id) { + @Transactional(REQUIRED) + public void onChange(Long id) { // default no-op } } diff --git a/src/main/java/io/debezium/platform/domain/LogStreamingService.java b/src/main/java/io/debezium/platform/domain/LogStreamingService.java new file mode 100644 index 00000000..8ce50a2b --- /dev/null +++ b/src/main/java/io/debezium/platform/domain/LogStreamingService.java @@ -0,0 +1,103 @@ +package io.debezium.platform.domain; + +import io.debezium.platform.environment.logs.LogReader; +import io.quarkus.virtual.threads.VirtualThreads; +import jakarta.enterprise.context.ApplicationScoped; +import lombok.Getter; +import org.jboss.logging.Logger; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Supplier; + +@ApplicationScoped +public class LogStreamingService { + + public static final int NO_DATA_SLEEP_MS = 1000; + + private final Logger logger; + private final ExecutorService executorService; + + public static class LogStreamingTask implements Runnable, Closeable { + private final Logger logger; + + @Getter + private final String name; + private final AtomicBoolean running; + private final Supplier supplier; + private final Consumer consumer; + + public LogStreamingTask(String name, Supplier supplier, Consumer consumer, Logger logger) { + this.name = name; + this.supplier = supplier; + this.consumer = consumer; + this.logger = logger; + this.running = new AtomicBoolean(false); + } + + public boolean isRunning() { + return running.get(); + } + + public void stop() { + if (!running.compareAndSet(true, false)) { + logger.infof("Stopping log streamer for '%s'", name); + } + } + + @Override + public void run() { + if (!running.compareAndSet(false, true)) { + return; + } + logger.infof("Starting log streamer for '%s'", name); + + try (var reader = supplier.get()) { + doStream(reader); + logger.infof("Finished streaming from log %s", name); + } catch (IOException e) { + logger.errorf("Error streaming from log %s", name); + } catch (InterruptedException e) { + logger.errorf("Interrupted while waiting for more logs from log %s", name); + Thread.currentThread().interrupt(); + } finally { + stop(); + } + } + + private void doStream(LogReader reader) throws InterruptedException, IOException { + while (isRunning()) { + var line = reader.readLine(); + if (line == null) { + Thread.sleep(NO_DATA_SLEEP_MS); + } + consumer.accept(line); + } + } + + @Override + public void close() { + stop(); + } + } + + public LogStreamingService(Logger logger, @VirtualThreads ExecutorService executorService) { + this.logger = logger; + this.executorService = executorService; + } + + /** + * Starts streaming the log, passing each line to given consumer. + * + * @param consumer log consumer + */ + public LogStreamingTask stream(String name, Supplier logSupplier, Consumer consumer) { + logger.infof("Starting log streamer for log %s", name); + var task = new LogStreamingTask(name, logSupplier, consumer, logger); + executorService.submit(task); + return task; + } +} diff --git a/src/main/java/io/debezium/platform/domain/PipelineService.java b/src/main/java/io/debezium/platform/domain/PipelineService.java index 0408b30b..d891bb11 100644 --- a/src/main/java/io/debezium/platform/domain/PipelineService.java +++ b/src/main/java/io/debezium/platform/domain/PipelineService.java @@ -9,34 +9,75 @@ import io.debezium.platform.domain.views.Pipeline; import io.debezium.platform.domain.views.flat.PipelineFlat; import io.debezium.platform.domain.views.refs.PipelineReference; +import io.debezium.platform.environment.EnvironmentController; import io.debezium.platform.environment.watcher.events.PipelineEvent; +import io.quarkus.arc.All; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Event; -import jakarta.inject.Inject; import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; + +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; @ApplicationScoped public class PipelineService extends AbstractService { - @Inject - Event> event; - - @Inject - ObjectMapper objectMapper; + private final Event> event; + private final ObjectMapper objectMapper; + private final LogStreamingService logStreamer; + private final List environmentControllers; - public PipelineService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewManager evm) { + public PipelineService(EntityManager em, + CriteriaBuilderFactory cbf, + EntityViewManager evm, + Event> event, + ObjectMapper objectMapper, + LogStreamingService logStreamer, + @All List environmentControllers) { super(PipelineEntity.class, Pipeline.class, PipelineReference.class, em, cbf, evm); + this.event = event; + this.objectMapper = objectMapper; + this.logStreamer = logStreamer; + this.environmentControllers = environmentControllers; } @Override - protected void onChange(Pipeline view) { - var flat =findByIdAs(PipelineFlat.class, view.getId()).orElseThrow(); + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Pipeline view) { + var flat = findByIdAs(PipelineFlat.class, view.getId()).orElseThrow(); event.fire(PipelineEvent.update(flat, objectMapper)); } @Override - protected void onChange(Long id) { + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Long id) { event.fire(PipelineEvent.delete(id)); } + + /** + * Returns the {@link EnvironmentController} instance for the given pipeline + * + * @param id pipeline id + * @return {@link EnvironmentController} instance for the given pipeline + */ + public Optional environmentController(Long id) { + // TODO: only operator environment is supported currently; + return findById(id).map(pipeline -> environmentControllers.getFirst()); + } + + /** + * Streams logs for the given pipeline, invoking given consumer for each log line + * + * @param id the pipeline id + * @param consumer the consumer to invoke for each log line + * @return log streaming task or empty optional if pipeline was not found + */ + public Optional streamLogs(Long id, Consumer consumer) { + return environmentController(id) + .map(EnvironmentController::pipelines) + .map(pipelines -> logStreamer.stream(String.valueOf(id), () -> pipelines.logReader(id), consumer)); + } } diff --git a/src/main/java/io/debezium/platform/domain/VaultService.java b/src/main/java/io/debezium/platform/domain/VaultService.java index e79f7ed9..73099b2c 100644 --- a/src/main/java/io/debezium/platform/domain/VaultService.java +++ b/src/main/java/io/debezium/platform/domain/VaultService.java @@ -13,8 +13,7 @@ import jakarta.enterprise.event.Event; import jakarta.inject.Inject; import jakarta.persistence.EntityManager; - -import static jakarta.transaction.Transactional.TxType.SUPPORTS; +import jakarta.transaction.Transactional; @ApplicationScoped @@ -31,12 +30,14 @@ public VaultService(EntityManager em, CriteriaBuilderFactory cbf, EntityViewMana } @Override - protected void onChange(Vault view) { + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Vault view) { event.fire(VaultEvent.update(view, objectMapper)); } @Override - protected void onChange(Long id) { + @Transactional(Transactional.TxType.REQUIRED) + public void onChange(Long id) { event.fire(VaultEvent.delete(id)); } } diff --git a/src/main/java/io/debezium/platform/environment/PipelineController.java b/src/main/java/io/debezium/platform/environment/PipelineController.java index ab1d37ed..15a63a9c 100644 --- a/src/main/java/io/debezium/platform/environment/PipelineController.java +++ b/src/main/java/io/debezium/platform/environment/PipelineController.java @@ -1,11 +1,55 @@ package io.debezium.platform.environment; import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.logs.LogReader; +/** + * Pipeline environment controller + */ public interface PipelineController { + /** + * Deploys the pipeline into target environment + *

+ * This method should never be called directly, instead rely on Outbox to + * guarantee the pipeline creation; + *

+ * + * @param pipeline the pipeline to deploy + */ void deploy(PipelineFlat pipeline); + /** + * Undeploys the pipeline with given id from target environment + *

+ * This method should never be called directly, instead rely on Outbox to + * guarantee the pipeline removal; + *

+ * + * @param id the pipeline id + */ void undeploy(Long id); + /** + * Stops the pipeline with given id + * + * @param id the pipeline id + */ + void stop(Long id); + + /** + * Starts the pipeline with given id + * + * @param id the pipeline id + */ + void start(Long id); + + /** + * Returns the {@link LogReader} instance for the given pipeline + * + * @param id the pipeline id + * @return {@link LogReader} instance for the given pipeline + */ + LogReader logReader(Long id); + } diff --git a/src/main/java/io/debezium/platform/environment/logs/LogReader.java b/src/main/java/io/debezium/platform/environment/logs/LogReader.java new file mode 100644 index 00000000..663418da --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/logs/LogReader.java @@ -0,0 +1,34 @@ +package io.debezium.platform.environment.logs; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.IOException; + +public interface LogReader extends Closeable { + + /** + * @return a {@link BufferedReader} that can be used to read the log + * @throws IOException if an I/O error occurs + */ + BufferedReader reader() throws IOException; + + /** + * Reads a single line from the log. + *

+ * This method is a shortcut for calling {@link #reader()} and then {@link BufferedReader#readLine()}. + *
+ * Note that if this method is called, the reader must be closed by calling {@link #close()}. + *

+ * + * @return the line read from the log, or {@code null} if the end of the stream has been reached + * @throws IOException if an I/O error occurs + */ + String readLine() throws IOException; + + /** + * Closes the log reader and releases any resources associated + * with it (e.g. {@link BufferedReader} returned by calling {@link #reader()}). + */ + @Override + void close() throws IOException; +} diff --git a/src/main/java/io/debezium/platform/environment/operator/DevClusterInitializer.java b/src/main/java/io/debezium/platform/environment/operator/DevClusterInitializer.java new file mode 100644 index 00000000..86049d61 --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/DevClusterInitializer.java @@ -0,0 +1,74 @@ +package io.debezium.platform.environment.operator; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.debezium.outbox.quarkus.ExportedEvent; +import io.debezium.platform.domain.PipelineService; +import io.debezium.platform.domain.views.flat.PipelineFlat; +import io.debezium.platform.environment.watcher.config.WatcherConfigGroup; +import io.debezium.platform.environment.watcher.events.PipelineEvent; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.quarkus.arc.profile.IfBuildProfile; +import io.quarkus.runtime.Startup; +import io.quarkus.runtime.StartupEvent; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Event; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import org.jboss.logging.Logger; + +import java.net.MalformedURLException; +import java.net.URI; +import java.util.Optional; + +@Startup +@ApplicationScoped +@IfBuildProfile("dev") +public class DevClusterInitializer { + + private final Logger logger; + private final KubernetesClient k8s; + private final WatcherConfigGroup watcherConfig; + private final PipelineService pipelineService; + private final Event> event; + private final ObjectMapper objectMapper; + + @Inject + EntityManager entityManager; + + public DevClusterInitializer(Logger logger, KubernetesClient k8s, + WatcherConfigGroup watcherConfig, PipelineService pipelineService, + Event> event, ObjectMapper objectMapper) { + this.logger = logger; + this.k8s = k8s; + this.watcherConfig = watcherConfig; + this.pipelineService = pipelineService; + this.event = event; + this.objectMapper = objectMapper; + } + + public void init(@Observes StartupEvent event) { + watcherConfig.crd().ifPresent(this::install); + initPipelines(); + } + + public void install(String crdUrl) { + try { + logger.info("Installing CRD from " + crdUrl); + var url = URI.create(crdUrl).toURL(); + var crds = k8s.apiextensions().v1().customResourceDefinitions(); + var crd = crds.load(url).item(); + + crds.resource(crd).serverSideApply(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + + public void initPipelines() { + logger.info("Firing pipeline update events for existing pipelines"); + pipelineService.list().forEach(pipelineService::onChange); + } +} diff --git a/src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java b/src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java deleted file mode 100644 index 75da6e91..00000000 --- a/src/main/java/io/debezium/platform/environment/operator/DevCrdInstaller.java +++ /dev/null @@ -1,46 +0,0 @@ -package io.debezium.platform.environment.operator; - -import io.debezium.platform.environment.watcher.config.WatcherConfigGroup; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.quarkus.arc.profile.IfBuildProfile; -import io.quarkus.runtime.Startup; -import jakarta.annotation.PostConstruct; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import org.jboss.logging.Logger; - -import java.net.MalformedURLException; -import java.net.URI; - -@Startup -@ApplicationScoped -@IfBuildProfile("dev") -public class DevCrdInstaller { - - Logger logger; - KubernetesClient k8s; - WatcherConfigGroup watcherConfig; - - public DevCrdInstaller(Logger logger, KubernetesClient k8s, WatcherConfigGroup watcherConfig) { - this.logger = logger; - this.k8s = k8s; - this.watcherConfig = watcherConfig; - } - - @PostConstruct - public void init() { - watcherConfig.crd().ifPresent(this::install); - } - - public void install(String crdUrl) { - try { - var url = URI.create(crdUrl).toURL(); - var crds = k8s.apiextensions().v1().customResourceDefinitions(); - var crd = crds.load(url).item(); - - crds.resource(crd).serverSideApply(); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java index 8669daee..cc31dd8f 100644 --- a/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorEnvironmentController.java @@ -4,10 +4,13 @@ import io.debezium.platform.environment.PipelineController; import io.debezium.platform.environment.VaultController; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Named; import org.jboss.logging.Logger; @ApplicationScoped +@Named(OperatorEnvironmentController.BEAN_NAME) public class OperatorEnvironmentController implements EnvironmentController { + public static final String BEAN_NAME = "operator-environment-controller"; protected final Logger logger; private final OperatorPipelineController pipelineController; diff --git a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java index 1299860d..d2f9052f 100644 --- a/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java +++ b/src/main/java/io/debezium/platform/environment/operator/OperatorPipelineController.java @@ -16,11 +16,16 @@ import io.debezium.operator.api.model.source.storage.schema.InMemorySchemaHistoryStore; import io.debezium.platform.domain.views.flat.PipelineFlat; import io.debezium.platform.environment.PipelineController; +import io.debezium.platform.environment.operator.logs.KubernetesLogReader; +import io.debezium.platform.environment.logs.LogReader; +import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.Loggable; import jakarta.enterprise.context.Dependent; import java.util.Map; +import java.util.Optional; @Dependent @@ -108,4 +113,44 @@ public void undeploy(Long id) { .withLabels(Map.of(LABEL_DBZ_CONDUCTOR_ID, id.toString())) .delete(); } + + @Override + public void stop(Long id) { + stop(id, true); + } + + @Override + public void start(Long id) { + stop(id, false); + } + + public Optional findById(Long id) { + return k8s.resources(DebeziumServer.class) + .withLabels(Map.of(LABEL_DBZ_CONDUCTOR_ID, id.toString())) + .list() + .getItems() + .stream() + .findFirst(); + } + + private Loggable findDeploymentLoggable(Long id, int tailingLines) { + return findById(id) + .map(DebeziumServer::getMetadata) + .map(ObjectMeta::getName) + .map(name -> k8s.apps().deployments().withName(name)) + .map(d -> d.tailingLines(tailingLines)) + .get(); + } + + @Override + public LogReader logReader(Long id) { + return new KubernetesLogReader(() -> findDeploymentLoggable(id, 100)); + } + + private void stop(Long id, boolean stop) { + findById(id).ifPresent(ds -> { + ds.setStopped(stop); + k8s.resource(ds).serverSideApply(); + }); + } } diff --git a/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java b/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java new file mode 100644 index 00000000..5576fcdf --- /dev/null +++ b/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java @@ -0,0 +1,55 @@ +package io.debezium.platform.environment.operator.logs; + +import io.debezium.platform.environment.logs.LogReader; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.dsl.LogWatch; +import io.fabric8.kubernetes.client.dsl.Loggable; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Objects; +import java.util.function.Supplier; + +public class KubernetesLogReader implements LogReader { + + private final Supplier supplier; + private LogWatch watch; + private BufferedReader reader; + + public KubernetesLogReader(Supplier supplier) { + this.supplier = Objects.requireNonNull(supplier, "Supplier cannot be null"); + } + + @Override + public BufferedReader reader() throws IOException { + return ensureReader(); + } + + @Override + public String readLine() throws IOException { + return reader().readLine(); + } + + @Override + public void close() throws IOException { + if (reader != null) { + reader.close(); + } + if (watch != null) { + watch.close(); + } + } + + private BufferedReader ensureReader() throws IOException { + if (reader == null) { + try { + this.watch = supplier.get().watchLog(); + this.reader = new BufferedReader(new InputStreamReader(watch.getOutput())); + } catch (KubernetesClientException e) { + throw new IOException(e); + } + } + return reader; + } +} diff --git a/src/main/java/io/debezium/platform/error/NotFoundException.java b/src/main/java/io/debezium/platform/error/NotFoundException.java new file mode 100644 index 00000000..ac5ee3b8 --- /dev/null +++ b/src/main/java/io/debezium/platform/error/NotFoundException.java @@ -0,0 +1,15 @@ +package io.debezium.platform.error; + +public class NotFoundException extends RuntimeException { + + private final long id; + + public NotFoundException(Long id) { + super("Invalid resource with id: " + id); + this.id = id; + } + + public long getId() { + return id; + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 2a5c6c52..772093c3 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -7,6 +7,9 @@ conductor: file: offsets.dat quarkus: + http: + cors: + ~: true debezium-outbox: table-name: events aggregate-type: @@ -40,6 +43,8 @@ quarkus: enabled: true port: 5432 image-name: quay.io/debezium/postgres:16-alpine + http: + port: 8081 log: level: INFO category: From ef3b077c66c9a34f211fafd40e2ad860f0a55171 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Wed, 26 Jun 2024 16:03:12 +0200 Subject: [PATCH 29/44] Fixed duplicit destination name in test --- src/test/java/io/debezium/platform/ServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/debezium/platform/ServiceTest.java b/src/test/java/io/debezium/platform/ServiceTest.java index 04e42311..f1e7aae5 100644 --- a/src/test/java/io/debezium/platform/ServiceTest.java +++ b/src/test/java/io/debezium/platform/ServiceTest.java @@ -104,7 +104,7 @@ public void createDestinations() { Assumptions.assumeThat(vaults).hasSize(2); var destination1 = evm.create(Destination.class); - destination1.setName("destination2"); + destination1.setName("destination1"); destination1.setSchema("schemaDXY"); destination1.setType("pubsub"); destination1.setVaults(Set.of(vaultRef(0),vaultRef(1))); From d794670e7e063bc9b7cb85b52ebf876e84e39f9f Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 9 Jul 2024 12:05:07 +0200 Subject: [PATCH 30/44] Updated operator api to 3.0.0-SNAPHSOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8e9c026f..af27da93 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ io.debezium debezium-operator-api - 2.7.0-SNAPSHOT + 3.0.0-SNAPSHOT io.quarkus From 0a840d724af1d090de99e04fe165061acf2df691 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 9 Jul 2024 12:11:28 +0200 Subject: [PATCH 31/44] Ability to access the entire pipeline log over REST --- .../platform/api/PipelineResource.java | 23 +++++++++++++++++++ .../platform/environment/logs/LogReader.java | 13 +++++++++++ .../operator/OperatorPipelineController.java | 7 +++--- .../operator/logs/KubernetesLogReader.java | 14 +++++++---- 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/debezium/platform/api/PipelineResource.java b/src/main/java/io/debezium/platform/api/PipelineResource.java index e7ca4e83..0c8b358f 100644 --- a/src/main/java/io/debezium/platform/api/PipelineResource.java +++ b/src/main/java/io/debezium/platform/api/PipelineResource.java @@ -3,6 +3,9 @@ import com.blazebit.persistence.integration.jaxrs.EntityViewId; import io.debezium.platform.domain.PipelineService; import io.debezium.platform.domain.views.Pipeline; +import io.debezium.platform.environment.EnvironmentController; +import io.debezium.platform.environment.logs.LogReader; +import io.smallrye.common.annotation.RunOnVirtualThread; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.DELETE; @@ -11,6 +14,7 @@ import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriInfo; @@ -28,6 +32,7 @@ import java.net.URI; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static jakarta.ws.rs.core.MediaType.TEXT_PLAIN; @Tag(name = "pipelines") @OpenAPIDefinition( @@ -107,4 +112,22 @@ public Response delete(@PathParam("id") Long id) { pipelineService.delete(id); return Response.status(Response.Status.NO_CONTENT).build(); } + + @Operation(summary = "Returns logs for pipeline with given id") + @APIResponse( + responseCode = "200", + content = @Content(mediaType = TEXT_PLAIN, schema = @Schema(implementation = String.class, required = true)) + ) + @GET + @Path("/{id}/logs") + @Produces(TEXT_PLAIN) + @RunOnVirtualThread + public Response getLogById(@PathParam("id") Long id) { + return pipelineService.environmentController(id) + .map(EnvironmentController::pipelines) + .map(pipelines -> pipelines.logReader(id)) + .map(LogReader::readAll) + .map(log -> Response.ok(log).build()) + .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); + } } diff --git a/src/main/java/io/debezium/platform/environment/logs/LogReader.java b/src/main/java/io/debezium/platform/environment/logs/LogReader.java index 663418da..0c1e343a 100644 --- a/src/main/java/io/debezium/platform/environment/logs/LogReader.java +++ b/src/main/java/io/debezium/platform/environment/logs/LogReader.java @@ -6,7 +6,20 @@ public interface LogReader extends Closeable { + + /** + * Reads the entire log content currently available. + * + * @return the log content + */ + String readAll(); + /** + * Obtains a {@link BufferedReader} that can be used to read live logs + *

+ * Note that if this method is called, the reader must be closed by calling {@link #close()}. + *

findById(Long id) { .findFirst(); } - private Loggable findDeploymentLoggable(Long id, int tailingLines) { + private TailPrettyLoggable findDeploymentLoggable(Long id) { return findById(id) .map(DebeziumServer::getMetadata) .map(ObjectMeta::getName) .map(name -> k8s.apps().deployments().withName(name)) - .map(d -> d.tailingLines(tailingLines)) .get(); } @Override public LogReader logReader(Long id) { - return new KubernetesLogReader(() -> findDeploymentLoggable(id, 100)); + return new KubernetesLogReader(() -> findDeploymentLoggable(id)); } private void stop(Long id, boolean stop) { diff --git a/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java b/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java index 5576fcdf..5a5f56e4 100644 --- a/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java +++ b/src/main/java/io/debezium/platform/environment/operator/logs/KubernetesLogReader.java @@ -3,7 +3,7 @@ import io.debezium.platform.environment.logs.LogReader; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.dsl.LogWatch; -import io.fabric8.kubernetes.client.dsl.Loggable; +import io.fabric8.kubernetes.client.dsl.TailPrettyLoggable; import java.io.BufferedReader; import java.io.IOException; @@ -13,14 +13,20 @@ public class KubernetesLogReader implements LogReader { - private final Supplier supplier; + public static final int STREAM_TAIL_LINES = 100; + + private final Supplier supplier; private LogWatch watch; private BufferedReader reader; - public KubernetesLogReader(Supplier supplier) { + public KubernetesLogReader(Supplier supplier) { this.supplier = Objects.requireNonNull(supplier, "Supplier cannot be null"); } + public String readAll() { + return supplier.get().getLog(); + } + @Override public BufferedReader reader() throws IOException { return ensureReader(); @@ -44,7 +50,7 @@ public void close() throws IOException { private BufferedReader ensureReader() throws IOException { if (reader == null) { try { - this.watch = supplier.get().watchLog(); + this.watch = supplier.get().tailingLines(STREAM_TAIL_LINES).watchLog(); this.reader = new BufferedReader(new InputStreamReader(watch.getOutput())); } catch (KubernetesClientException e) { throw new IOException(e); From 19f637f71fdcbf73e32021a2a5097c367e84d02e Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 9 Jul 2024 12:15:29 +0200 Subject: [PATCH 32/44] Added simple REST seeding script --- examples/compose-kind-kafka/seed.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 examples/compose-kind-kafka/seed.sh diff --git a/examples/compose-kind-kafka/seed.sh b/examples/compose-kind-kafka/seed.sh new file mode 100755 index 00000000..807ae151 --- /dev/null +++ b/examples/compose-kind-kafka/seed.sh @@ -0,0 +1,4 @@ +port=${1:-8080} +http POST localhost:${port}/api/sources @payloads/source.json +http POST localhost:${port}/api/destinations @payloads/destination.json +http POST localhost:${port}/api/pipelines @payloads/pipeline.json \ No newline at end of file From 2dafeffd5d3a7f301862fb6993cfb4a21b1ffc13 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 9 Jul 2024 12:34:42 +0200 Subject: [PATCH 33/44] Full pipeline log is now served as downloadable file --- src/main/java/io/debezium/platform/api/PipelineResource.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/debezium/platform/api/PipelineResource.java b/src/main/java/io/debezium/platform/api/PipelineResource.java index 0c8b358f..752c835c 100644 --- a/src/main/java/io/debezium/platform/api/PipelineResource.java +++ b/src/main/java/io/debezium/platform/api/PipelineResource.java @@ -127,7 +127,9 @@ public Response getLogById(@PathParam("id") Long id) { .map(EnvironmentController::pipelines) .map(pipelines -> pipelines.logReader(id)) .map(LogReader::readAll) - .map(log -> Response.ok(log).build()) + .map(log -> Response.ok(log) + .header("Content-Disposition", "attachment; filename=pipeline.log") + .build()) .orElseGet(() -> Response.status(Response.Status.NOT_FOUND).build()); } } From 2f891db64e525e1d23fa615c67096518d4f042f1 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Tue, 23 Jul 2024 09:40:25 +0200 Subject: [PATCH 34/44] Added explicit service dependnecies to compose --- examples/compose-kind-kafka/compose.yml | 4 ++++ examples/compose-rest-api-only/compose.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/examples/compose-kind-kafka/compose.yml b/examples/compose-kind-kafka/compose.yml index 854c4d85..4cc91342 100644 --- a/examples/compose-kind-kafka/compose.yml +++ b/examples/compose-kind-kafka/compose.yml @@ -2,6 +2,8 @@ services: conductor: image: quay.io/debezium/platform-conductor:nightly pull_policy: always + depends_on: + - postgres ports: - "8080:8080" environment: @@ -32,6 +34,8 @@ services: container_name: stage image: quay.io/debezium/platform-stage:latest pull_policy: always + depends_on: + - conductor ports: - "3000:3000" networks: diff --git a/examples/compose-rest-api-only/compose.yml b/examples/compose-rest-api-only/compose.yml index 98cbbef0..7ee9a681 100644 --- a/examples/compose-rest-api-only/compose.yml +++ b/examples/compose-rest-api-only/compose.yml @@ -2,6 +2,8 @@ services: conductor: image: quay.io/debezium/platform-conductor:nightly pull_policy: always + depends_on: + - postgres ports: - "8080:8080" environment: @@ -18,6 +20,8 @@ services: POSTGRES_PASSWORD: conductor POSTGRES_DB: conductor stage: + depends_on: + - conductor container_name: stage image: quay.io/debezium/platform-stage:latest pull_policy: always From 8672fda650729f5938ce95eb9b3296d7ff68c163 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 29 Jul 2024 09:55:53 +0200 Subject: [PATCH 35/44] Using DBZ operator 2.7.0-final in the example --- examples/compose-kind-kafka/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/compose-kind-kafka/env.sh b/examples/compose-kind-kafka/env.sh index 1667a13b..1d4dd525 100644 --- a/examples/compose-kind-kafka/env.sh +++ b/examples/compose-kind-kafka/env.sh @@ -3,7 +3,7 @@ NAMESPACE=debezium TIMEOUT=720s DBZ_INSTALL_OPERATOR=false -DBZ_OPERATOR_VERSION=2.7.0-beta2 +DBZ_OPERATOR_VERSION=2.7.0-final STRIMZI_INSTALL_OPERATOR=true STRIMZI_OPERATOR_VERSION=0.40.0 From 4d9be1395aeeefec34d1e98ab10b2cd1ff902fed Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 29 Jul 2024 09:56:33 +0200 Subject: [PATCH 36/44] Extended project information in README.md --- README.md | 65 +++++++++++++++++++++++++- img/debezium-platform-architecture.svg | 8 ++++ 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 img/debezium-platform-architecture.svg diff --git a/README.md b/README.md index bed6bf75..267d18a8 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,72 @@ # debezium-platform-conductor The back-end component of Debezium management platform. Conductor provides a set APIs which can be -used to orchestrate and control Debezium deployments. It's intended to be interacted with through a front-end client. +used to orchestrate and control Debezium deployments. Usually it's intended to be interacted with through a front-end client. +**Disclaimer**: This project is still in early development stage and should not be used in production. ## Debezium Management Platform Debezium Management Platform (Debezium Orchestra) aims to provide means to simplify the deployment of Debezium to various environments in highly opinionated manner. The goal is not to provide total control over environment specific configuration. To achieve this goal the platform uses -a data-centric view on Debezium components. \ No newline at end of file +a data-centric view on Debezium components. + + +## Platform Architecture +The platform is composed of two main components: + +1. Conductor: The back-end component which provides a set of APIs to orchestrate and control Debezium deployments. +2. Stage: The front-end component which provides a user interface to interact with the Conductor. + + +### Conductor Architecture +The conductor component itself is composed of several subcomponents: + +1. API Server: The main entry point for the platform. It provides a set of APIs to interact with the platform. +2. Watcher: Component responsible for the actual communication with deployment environment (e.g. Debezium Operator in K8s cluster). + + +![Debezium Management Platform Architecture](img/debezium-platform-architecture.svg) + +## How to Try the Platform +There are currently two examples available to try the platform under `examples` directory: + +1. `compose-rest-api-only`: This example provides a mean to run the API server and the front-end Stage application locally. While data is persisted in a local database, the platform does not interact with any deployment environment (which means no Debezium pipelines are actually deployed). +2. `compose-kind-kafka`: This example uses a more complex environment which relies on local Kubernetes cluster provision via [Kind](https://kind.sigs.k8s.io/) + +### Running the `compose-rest-api-only` Example +To run the example, execute the following command from the `examples/compose-rest-api-only` directory: + +```shell +# Using podman +podman compose -f compose.yml up + +# Using docker +docker compose -f compose.yml up +``` + +### Running the `compose-kind-kafka` Example +To run the example, execute the following commands from the `examples/compose-kind-kafka` directory: + +```shell +# Create the local Kubernetes cluster +./create-cluster.sh + +# Deploy the platform (podman) +docker compose -f compose.yml up + +# Deploy the platform (docker) +docker compose -f compose.yml up +``` + +The `create-cluster.sh` performs several steps: + +1. Creates a local Kubernetes cluster using Kind. +2. Exports "internal" `kubeconfig` file, which is then mounted to the conductor container. +3. Deploys Debezium Operator to the cluster. +4. Deploys Strimzi Operator to the cluster. +5. Provisions PostgreSQL database and Kafka Cluster in the cluster. + +The behaviour of `create-cluster.sh` can be modified by changing the values in the `env.sh` file. + + diff --git a/img/debezium-platform-architecture.svg b/img/debezium-platform-architecture.svg new file mode 100644 index 00000000..e2de7ab5 --- /dev/null +++ b/img/debezium-platform-architecture.svg @@ -0,0 +1,8 @@ + + + + + + + + CONDUCTOR-WATCHERDBZ EngineREST APIENVCONTROLLERDBZ OPERATORMongo_PubSub_PipelinePostgres_Kafka_PipelineCRMSGKUBERNETESSECRETOutboxMSGC1C2C3CONDUCTOR-APISERVICESUSERNext Back+? Source? SinkExtract New StateTransform DatePipeline CompositionCONDUCTORSTAGEUI \ No newline at end of file From c9f7fd9909c719db47dfb575764ddf1766cdacad Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Mon, 2 Sep 2024 11:51:18 +0200 Subject: [PATCH 37/44] [hotfix] K8s example should deploy DBZ operator by default --- examples/compose-kind-kafka/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/compose-kind-kafka/env.sh b/examples/compose-kind-kafka/env.sh index 1d4dd525..1c062477 100644 --- a/examples/compose-kind-kafka/env.sh +++ b/examples/compose-kind-kafka/env.sh @@ -2,7 +2,7 @@ CLUSTER=debezium NAMESPACE=debezium TIMEOUT=720s -DBZ_INSTALL_OPERATOR=false +DBZ_INSTALL_OPERATOR=true DBZ_OPERATOR_VERSION=2.7.0-final STRIMZI_INSTALL_OPERATOR=true From 48f4bfe08e140207a343d45f40575979878efca2 Mon Sep 17 00:00:00 2001 From: Jakub Cechacek Date: Thu, 7 Nov 2024 10:45:17 +0100 Subject: [PATCH 38/44] [examples] Updated full deployment example --- examples/compose-kind-kafka/create-cluster.sh | 2 ++ examples/compose-kind-kafka/env.sh | 4 +-- .../k8s/kafka/001_kafka.yml | 33 ++++++++++++++----- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/examples/compose-kind-kafka/create-cluster.sh b/examples/compose-kind-kafka/create-cluster.sh index 38d44745..d4d466bf 100755 --- a/examples/compose-kind-kafka/create-cluster.sh +++ b/examples/compose-kind-kafka/create-cluster.sh @@ -17,6 +17,7 @@ if $DBZ_INSTALL_OPERATOR; then echo ">>> Installing Debezium operator" echo ">> Add helm repo" helm repo add debezium https://charts.debezium.io + helm repo update debezium echo ">> Deploy operator" helm install debezium-operator debezium/debezium-operator --version $DBZ_OPERATOR_VERSION --namespace $NAMESPACE fi @@ -25,6 +26,7 @@ if $STRIMZI_INSTALL_OPERATOR; then echo ">>> Installing Strimzi operator" echo ">> Add helm repo" helm repo add strimzi https://strimzi.io/charts/ + helm repo update strimzi echo ">> Deploy operator" helm install strimzi-operator strimzi/strimzi-kafka-operator --version $STRIMZI_OPERATOR_VERSION --namespace $NAMESPACE fi diff --git a/examples/compose-kind-kafka/env.sh b/examples/compose-kind-kafka/env.sh index 1c062477..12dc988d 100644 --- a/examples/compose-kind-kafka/env.sh +++ b/examples/compose-kind-kafka/env.sh @@ -3,10 +3,10 @@ NAMESPACE=debezium TIMEOUT=720s DBZ_INSTALL_OPERATOR=true -DBZ_OPERATOR_VERSION=2.7.0-final +DBZ_OPERATOR_VERSION=3.0.0-final STRIMZI_INSTALL_OPERATOR=true -STRIMZI_OPERATOR_VERSION=0.40.0 +STRIMZI_OPERATOR_VERSION=0.44.0 STRIMZI_DEPLOY_KAFKA=true POSTGRES_DEPLOY=true diff --git a/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml b/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml index 25bc69bd..6fe30ba9 100644 --- a/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml +++ b/examples/compose-kind-kafka/k8s/kafka/001_kafka.yml @@ -1,11 +1,33 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaNodePool +metadata: + name: dual-role + labels: + strimzi.io/cluster: dbz-kafka +spec: + replicas: 1 + roles: + - controller + - broker + storage: + type: jbod + volumes: + - id: 0 + type: ephemeral + kraftMetadata: shared +--- + apiVersion: kafka.strimzi.io/v1beta2 kind: Kafka metadata: name: dbz-kafka + annotations: + strimzi.io/node-pools: enabled + strimzi.io/kraft: enabled spec: kafka: - version: 3.7.0 - replicas: 1 + version: 3.8.0 + metadataVersion: 3.8-IV0 listeners: - name: plain port: 9092 @@ -21,13 +43,6 @@ spec: transaction.state.log.min.isr: 1 default.replication.factor: 1 min.insync.replicas: 1 - inter.broker.protocol.version: "3.7" - storage: - type: ephemeral - zookeeper: - replicas: 1 - storage: - type: ephemeral entityOperator: topicOperator: {} userOperator: {} \ No newline at end of file From 4892f64145f7b62cb9ee149d334b8c703d1f1498 Mon Sep 17 00:00:00 2001 From: indraraj Date: Tue, 26 Nov 2024 19:11:01 +0530 Subject: [PATCH 39/44] DBZ-8419: Update the platform-stage env variable --- examples/compose-kind-kafka/compose.yml | 2 ++ examples/compose-rest-api-only/compose.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/compose-kind-kafka/compose.yml b/examples/compose-kind-kafka/compose.yml index 4cc91342..114f8fe9 100644 --- a/examples/compose-kind-kafka/compose.yml +++ b/examples/compose-kind-kafka/compose.yml @@ -38,6 +38,8 @@ services: - conductor ports: - "3000:3000" + environment: + CONDUCTOR_URL: http://localhost:8080 networks: - conductor volumes: diff --git a/examples/compose-rest-api-only/compose.yml b/examples/compose-rest-api-only/compose.yml index 7ee9a681..c1a97a5b 100644 --- a/examples/compose-rest-api-only/compose.yml +++ b/examples/compose-rest-api-only/compose.yml @@ -26,4 +26,6 @@ services: image: quay.io/debezium/platform-stage:latest pull_policy: always ports: - - "3000:3000" \ No newline at end of file + - "3000:3000" + environment: + CONDUCTOR_URL: http://localhost:8080 \ No newline at end of file From 503a9738bdb90812ef765b1e5960b7ee82abe9e3 Mon Sep 17 00:00:00 2001 From: Fiore Mario Vitale Date: Wed, 22 Jan 2025 14:20:02 +0100 Subject: [PATCH 40/44] Update README.md Archive note --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 267d18a8..f0117982 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +This repository has been archived infavour of [debezium-platform](https://github.com/debezium/debezium-platform) + + # debezium-platform-conductor The back-end component of Debezium management platform. Conductor provides a set APIs which can be From 2dda383e7b68fd66cbe36c1375f58a2d41ed1943 Mon Sep 17 00:00:00 2001 From: Bhawani Shanker Sharma Date: Thu, 23 Apr 2026 22:11:27 +0530 Subject: [PATCH 41/44] debezium/dbz#1851 Add imagePullSecrets and extend imagePullPolicy to all chart images Signed-off-by: Bhawani Shanker Sharma --- helm/README.md | 7 +++++++ helm/templates/conductor-deployment.yaml | 9 ++++++++- helm/templates/stage-deployment.yaml | 9 ++++++++- helm/values.yaml | 22 ++++++++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/helm/README.md b/helm/README.md index c63a5c1b..66ff2294 100644 --- a/helm/README.md +++ b/helm/README.md @@ -60,6 +60,13 @@ you `values.yaml` with your domain. | schemaHistory.database.auth.username | Database username | user | | schemaHistory.database.auth.password | Database password | password | | | | | env | List of env variable to pass to the conductor | [] | +| imagePullSecrets | Global list of imagePullSecrets applied to all container images. Per-component `imagePullSecrets` override this value (not merged). | [] | +| stage.imagePullSecrets | Overrides global `imagePullSecrets` for the stage image. | [] | +| conductor.imagePullSecrets | Overrides global `imagePullSecrets` for the conductor image. | [] | +| conductor.descriptors.official.imagePullPolicy | Image pull policy for the descriptors OCI image. | IfNotPresent | +| conductor.descriptors.official.imagePullSecrets | Overrides global `imagePullSecrets` for the descriptors image. | [] | +| server.imagePullPolicy | Image pull policy for Debezium Server instances created by pipelines. | IfNotPresent | +| server.imagePullSecrets | Overrides global `imagePullSecrets` for the Debezium Server image. | [] | ## Descriptor OCI Artifacts diff --git a/helm/templates/conductor-deployment.yaml b/helm/templates/conductor-deployment.yaml index d8675de6..ac48ff8a 100644 --- a/helm/templates/conductor-deployment.yaml +++ b/helm/templates/conductor-deployment.yaml @@ -22,6 +22,13 @@ spec: app.kubernetes.io/name: conductor spec: serviceAccountName: conductor-service-account +{{- $conductorSecrets := .Values.conductor.imagePullSecrets | default list }} +{{- $globalSecrets := .Values.imagePullSecrets | default list }} +{{- $pullSecrets := ternary $conductorSecrets $globalSecrets (gt (len $conductorSecrets) 0) }} +{{- if gt (len $pullSecrets) 0 }} + imagePullSecrets: + {{- toYaml $pullSecrets | nindent 8 }} +{{- end }} containers: - name: conductor image: {{ .Values.conductor.image }} @@ -73,4 +80,4 @@ spec: - name: {{ .name }} value: {{ .value }} {{- end }} - restartPolicy: Always + restartPolicy: Always \ No newline at end of file diff --git a/helm/templates/stage-deployment.yaml b/helm/templates/stage-deployment.yaml index 41a1c50d..5c3693eb 100644 --- a/helm/templates/stage-deployment.yaml +++ b/helm/templates/stage-deployment.yaml @@ -19,6 +19,13 @@ spec: labels: app.kubernetes.io/name: stage spec: +{{- $stageSecrets := .Values.stage.imagePullSecrets | default list }} +{{- $globalSecrets := .Values.imagePullSecrets | default list }} +{{- $pullSecrets := ternary $stageSecrets $globalSecrets (gt (len $stageSecrets) 0) }} +{{- if gt (len $pullSecrets) 0 }} + imagePullSecrets: + {{- toYaml $pullSecrets | nindent 8 }} +{{- end }} containers: - image: {{ .Values.stage.image }} imagePullPolicy: {{ .Values.stage.imagePullPolicy }} @@ -29,4 +36,4 @@ spec: env: - name: CONDUCTOR_URL value: {{ required "A valid domain URL required!" (include "debezium-platform.domainUrl" .) }} - restartPolicy: Always + restartPolicy: Always \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 19e496cb..21626e67 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,12 +1,25 @@ domain: url: "" name: "" + +# Global imagePullSecrets applied to all container images by default. +# Per-component imagePullSecrets override this value (not merged). +# Example: +# imagePullSecrets: +# - name: my-registry-secret +imagePullSecrets: [] + stage: image: quay.io/debezium/platform-stage:nightly imagePullPolicy: IfNotPresent + # Overrides global imagePullSecrets for the stage image. + imagePullSecrets: [] + conductor: image: quay.io/debezium/platform-conductor:nightly imagePullPolicy: IfNotPresent + # Overrides global imagePullSecrets for the conductor image. + imagePullSecrets: [] offset: existingConfigMap: "" descriptors: @@ -16,8 +29,17 @@ conductor: image: debezium/debezium-descriptors tag: nightly mountPath: /opt/descriptors + # Image pull policy for the descriptors OCI image. + imagePullPolicy: IfNotPresent + # Overrides global imagePullSecrets for the descriptors image. + imagePullSecrets: [] + server: image: "" + # Image pull policy for Debezium Server instances. + imagePullPolicy: IfNotPresent + # Overrides global imagePullSecrets for the Debezium Server image. + imagePullSecrets: [] ingress: enabled: true className: "" From e336c9935ba9037cd8096f7d40075b407371d633 Mon Sep 17 00:00:00 2001 From: Bhawani Shanker Sharma Date: Wed, 29 Apr 2026 23:38:32 +0530 Subject: [PATCH 42/44] debezium/dbz#1851 Address review comments: use coalesce for imagePullSecrets, add ORAS credentials via secretKeyRef, remove unsupported imagePullPolicy from server Signed-off-by: Bhawani Shanker Sharma --- .../src/main/resources/application.yml | 3 ++- helm/templates/conductor-deployment.yaml | 16 +++++++++++++--- helm/templates/stage-deployment.yaml | 4 +--- helm/values.yaml | 14 +++++++------- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/debezium-platform-conductor/src/main/resources/application.yml b/debezium-platform-conductor/src/main/resources/application.yml index 70e80da9..d16d9e35 100644 --- a/debezium-platform-conductor/src/main/resources/application.yml +++ b/debezium-platform-conductor/src/main/resources/application.yml @@ -71,7 +71,6 @@ destinations: pulsar: connection: timeout: 60 - rabbitmq: connection: timeout: 60 @@ -123,6 +122,8 @@ quarkus: "registry": host: ${conductor.descriptors.image.registry} enabled: true + username: ${CONDUCTOR_DESCRIPTORS_REGISTRY_USERNAME:} + password: ${CONDUCTOR_DESCRIPTORS_REGISTRY_PASSWORD:} "%dev": conductor: diff --git a/helm/templates/conductor-deployment.yaml b/helm/templates/conductor-deployment.yaml index ac48ff8a..25aedd47 100644 --- a/helm/templates/conductor-deployment.yaml +++ b/helm/templates/conductor-deployment.yaml @@ -22,9 +22,7 @@ spec: app.kubernetes.io/name: conductor spec: serviceAccountName: conductor-service-account -{{- $conductorSecrets := .Values.conductor.imagePullSecrets | default list }} -{{- $globalSecrets := .Values.imagePullSecrets | default list }} -{{- $pullSecrets := ternary $conductorSecrets $globalSecrets (gt (len $conductorSecrets) 0) }} +{{- $pullSecrets := coalesce .Values.conductor.imagePullSecrets .Values.imagePullSecrets | default list }} {{- if gt (len $pullSecrets) 0 }} imagePullSecrets: {{- toYaml $pullSecrets | nindent 8 }} @@ -69,6 +67,18 @@ spec: value: {{ .Values.conductor.descriptors.official.image }} - name: CONDUCTOR_DESCRIPTORS_IMAGE_TAG value: {{ .Values.conductor.descriptors.official.tag }} +{{- if and .Values.conductor.descriptors.official.auth .Values.conductor.descriptors.official.auth.existingSecret }} + - name: CONDUCTOR_DESCRIPTORS_REGISTRY_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.conductor.descriptors.official.auth.existingSecret }} + key: username + - name: CONDUCTOR_DESCRIPTORS_REGISTRY_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.conductor.descriptors.official.auth.existingSecret }} + key: password +{{- end }} {{- end }} {{- if .Values.server.image }} - name: PIPELINE_SERVER_IMAGE diff --git a/helm/templates/stage-deployment.yaml b/helm/templates/stage-deployment.yaml index 5c3693eb..89fde450 100644 --- a/helm/templates/stage-deployment.yaml +++ b/helm/templates/stage-deployment.yaml @@ -19,9 +19,7 @@ spec: labels: app.kubernetes.io/name: stage spec: -{{- $stageSecrets := .Values.stage.imagePullSecrets | default list }} -{{- $globalSecrets := .Values.imagePullSecrets | default list }} -{{- $pullSecrets := ternary $stageSecrets $globalSecrets (gt (len $stageSecrets) 0) }} +{{- $pullSecrets := coalesce .Values.stage.imagePullSecrets .Values.imagePullSecrets | default list }} {{- if gt (len $pullSecrets) 0 }} imagePullSecrets: {{- toYaml $pullSecrets | nindent 8 }} diff --git a/helm/values.yaml b/helm/values.yaml index 21626e67..eaff9dfe 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -29,17 +29,17 @@ conductor: image: debezium/debezium-descriptors tag: nightly mountPath: /opt/descriptors - # Image pull policy for the descriptors OCI image. - imagePullPolicy: IfNotPresent - # Overrides global imagePullSecrets for the descriptors image. - imagePullSecrets: [] + # imagePullPolicy removed - not applicable for ORAS-based pulls + # imagePullSecrets replaced by auth.existingSecret for ORAS credentials + auth: + existingSecret: "" # Secret must contain keys: username, password server: image: "" - # Image pull policy for Debezium Server instances. - imagePullPolicy: IfNotPresent - # Overrides global imagePullSecrets for the Debezium Server image. + # imagePullPolicy is not yet supported by DebeziumOperator. + # Tracked in: https://github.com/debezium/debezium/issues/1851 imagePullSecrets: [] + ingress: enabled: true className: "" From 1c93f7017f32550a1463c757f9a9095d1a64bbc0 Mon Sep 17 00:00:00 2001 From: Bhawani Shanker Sharma Date: Fri, 15 May 2026 11:23:14 +0530 Subject: [PATCH 43/44] debezium/dbz#1851 Add imagePullSecrets and extend imagePullPolicy to all chart images Signed-off-by: Bhawani Shanker Sharma --- helm/README.md | 128 +++++++++++------------ helm/templates/conductor-deployment.yaml | 6 +- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/helm/README.md b/helm/README.md index 66ff2294..7925b137 100644 --- a/helm/README.md +++ b/helm/README.md @@ -4,73 +4,70 @@ This chart will install the components required to run the Debezium Platform. 2. Stage: The front-end component which provides a user interface to interact with the Conductor. 3. Debezium operator: operator that manages the creation of Debezium Server custom resource. 4. [Optional] PostgreSQL database used by conductor to store its data. -5. [Optional] Strimzi operator: operator for creating Kakfa cluster. In case you want to use a Kafka destination in you - pipeline. +5. [Optional] Strimzi operator: operator for creating Kafka cluster. In case you want to use a Kafka destination in your pipeline. # Prerequisites -The chart use an ingress to expose `debezium-stage (UI)` and `debezium-conductor (backend)`, -this will require to have -an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) installed in you -cluster. -You need also to have domain that must point to the cluster IP and then configure the `domain.name` property in -you `values.yaml` with your domain. +The chart use an ingress to expose `debezium-stage (UI)` and `debezium-conductor (backend)`, this will require to have an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) installed in your cluster. + +You need also to have a domain that must point to the cluster IP and then configure the `domain.name` property in your `values.yaml` with your domain. ### Configurations -| Name | Description | Default | -|:-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| domain.name | domain used as ingress host | "" | -| domain.url | domain used as ingress host (DEPRECATED). Use `domain.name` instead.) | "" | -| ingress.enabled | Enable ingress resource for conductor/stage | true | -| ingress.className | Optional ingress class name | "" | -| ingress.annotations | Extra ingress annotations | {} | -| ingress.tls.enabled | Enable TLS section on ingress | false | -| ingress.tls.secretName | Secret name used when TLS is enabled | "" | -| stage.image | Image for the stage (UI) | quay.io/debezium/platform-stage:latest | -| stage.imagePullPolicy | Image pull policy for the stage container (UI). If empty it will default to IfNotPresent. | IfNotPresent | -| conductor.image | Image for the conductor | quay.io/debezium/platform-conductor:latest | -| conductor.imagePullPolicy | Image pull policy for the conductor container. If empty it will default to IfNotPresent. | IfNotPresent | -| conductor.offset.existingConfigMap | Name of the config map used to store conductor offsets. If empty it will be automatically created. | "" | -| conductor.descriptors.official.enabled | Enable official Debezium descriptors (downloaded via ORAS at startup) | true | -| conductor.descriptors.official.registry | Registry hosting the descriptor OCI artifact | quay.io | -| conductor.descriptors.official.image | Image name for the descriptor OCI artifact | debezium/debezium-descriptors | -| conductor.descriptors.official.tag | Image tag for the descriptor OCI artifact | nightly | -| conductor.descriptors.official.mountPath | Path where descriptors will be downloaded inside the container | /opt/descriptors | -| server.image | Image for Debezium Server instances created by pipelines. If empty, the operator's ServerImageProvider determines the image | "" | -| database.enabled | Enable the installation of PostgreSQL by the chart | false | -| database.name | Database name | postgres | -| database.host | Database host | postgres | -| database.auth.existingSecret | Name of the secret to where `username` and `password` are stored. If empty a secret will be created using the `username` and `password` properties | "" | -| database.auth.username | Database username | user | -| database.auth.password | Database password | password | -| debezium-operator.enabled | Enable the installation of the debezium-operator by the chart | true | -| offset.reusePlatformDatabase | Pipelines will use database to store offsets. By default the database used by the conductor service is used.
If you want to use a dedicated one set this property to false | true | -| offset.database.name | Database name | postgres | -| offset.database.host | Database host | postgres | -| offset.database.port | Database port | 5432 | | | -| offset.database.auth.existingSecret | Name of the secret to where `username` and `password` are stored. If not set `offset.database.auth.username` and `offset.database.auth.password` will be used. | "" | -| offset.database.auth.username | Database username | user | -| offset.database.auth.password | Database password | password | | | -| schemaHistory.reusePlatformDatabase | Pipelines will use database to store schema history. By default the database used by the conductor service is used.
If you want to use a dedicated one set this property to false | true | -| schemaHistory.database.name | Database name | postgres | -| schemaHistory.database.host | Database host | postgres | -| schemaHistory.database.port | Database port | 5432 | | | -| schemaHistory.database.auth.existingSecret | Name of the secret to where `username` and `password` are stored. If not set `schemaHistory.database.auth.username` and `schemaHistory.database.auth.password` will be used. | "" | -| schemaHistory.database.auth.username | Database username | user | -| schemaHistory.database.auth.password | Database password | password | | | | -| env | List of env variable to pass to the conductor | [] | -| imagePullSecrets | Global list of imagePullSecrets applied to all container images. Per-component `imagePullSecrets` override this value (not merged). | [] | -| stage.imagePullSecrets | Overrides global `imagePullSecrets` for the stage image. | [] | -| conductor.imagePullSecrets | Overrides global `imagePullSecrets` for the conductor image. | [] | -| conductor.descriptors.official.imagePullPolicy | Image pull policy for the descriptors OCI image. | IfNotPresent | -| conductor.descriptors.official.imagePullSecrets | Overrides global `imagePullSecrets` for the descriptors image. | [] | -| server.imagePullPolicy | Image pull policy for Debezium Server instances created by pipelines. | IfNotPresent | -| server.imagePullSecrets | Overrides global `imagePullSecrets` for the Debezium Server image. | [] | +| Name | Description | Default | +|:---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| +| domain.name | domain used as ingress host | "" | +| domain.url | domain used as ingress host (DEPRECATED). Use `domain.name` instead. | "" | +| ingress.enabled | Enable ingress resource for conductor/stage | true | +| ingress.className | Optional ingress class name | "" | +| ingress.annotations | Extra ingress annotations | {} | +| ingress.tls.enabled | Enable TLS section on ingress | false | +| ingress.tls.secretName | Secret name used when TLS is enabled | "" | +| stage.image | Image for the stage (UI) | quay.io/debezium/platform-stage:latest | +| stage.imagePullPolicy | Image pull policy for the stage container (UI). If empty it will default to IfNotPresent. | IfNotPresent | +| conductor.image | Image for the conductor | quay.io/debezium/platform-conductor:latest | +| conductor.imagePullPolicy | Image pull policy for the conductor container. If empty it will default to IfNotPresent. | IfNotPresent | +| conductor.offset.existingConfigMap | Name of the config map used to store conductor offsets. If empty it will be automatically created. | "" | +| conductor.descriptors.official.enabled | Enable official Debezium descriptors (downloaded via ORAS at startup) | true | +| conductor.descriptors.official.registry | Registry hosting the descriptor OCI artifact | quay.io | +| conductor.descriptors.official.image | Image name for the descriptor OCI artifact | debezium/debezium-descriptors | +| conductor.descriptors.official.tag | Image tag for the descriptor OCI artifact | nightly | +| conductor.descriptors.official.mountPath | Path where descriptors will be downloaded inside the container | /opt/descriptors | +| server.image | Image for Debezium Server instances created by pipelines. If empty, the operator's ServerImageProvider determines the image | "" | +| database.enabled | Enable the installation of PostgreSQL by the chart | false | +| database.name | Database name | postgres | +| database.host | Database host | postgres | +| database.auth.existingSecret | Name of the secret where `username` and `password` are stored. If empty a secret will be created using the `username` and `password` properties | "" | +| database.auth.username | Database username | user | +| database.auth.password | Database password | password | +| debezium-operator.enabled | Enable the installation of the debezium-operator by the chart | true | +| offset.reusePlatformDatabase | Pipelines will use database to store offsets. By default the database used by the conductor service is used. If you want to use a dedicated one set this property to false | true | +| offset.database.name | Database name | postgres | +| offset.database.host | Database host | postgres | +| offset.database.port | Database port | 5432 | +| offset.database.auth.existingSecret | Name of the secret where `username` and `password` are stored. If not set `offset.database.auth.username` and `offset.database.auth.password` will be used. | "" | +| offset.database.auth.username | Database username | user | +| offset.database.auth.password | Database password | password | +| schemaHistory.reusePlatformDatabase | Pipelines will use database to store schema history. By default the database used by the conductor service is used. If you want to use a dedicated one set this property to false | true | +| schemaHistory.database.name | Database name | postgres | +| schemaHistory.database.host | Database host | postgres | +| schemaHistory.database.port | Database port | 5432 | +| schemaHistory.database.auth.existingSecret | Name of the secret where `username` and `password` are stored. If not set `schemaHistory.database.auth.username` and `schemaHistory.database.auth.password` will be used. | "" | +| schemaHistory.database.auth.username | Database username | user | +| schemaHistory.database.auth.password | Database password | password | +| env | List of env variable to pass to the conductor | [] | +| imagePullSecrets | Global list of imagePullSecrets applied to all container images. Per-component `imagePullSecrets` override this value (not merged). | [] | +| stage.imagePullSecrets | Overrides global `imagePullSecrets` for the stage image. | [] | +| conductor.imagePullSecrets | Overrides global `imagePullSecrets` for the conductor image. | [] | +| conductor.descriptors.official.imagePullPolicy | Image pull policy for the descriptors OCI image. | IfNotPresent | +| conductor.descriptors.official.imagePullSecrets | Overrides global `imagePullSecrets` for the descriptors image. | [] | +| server.imagePullPolicy | Image pull policy for Debezium Server instances created by pipelines. | IfNotPresent | +| server.imagePullSecrets | Overrides global `imagePullSecrets` for the Debezium Server image. | [] | ## Descriptor OCI Artifacts The conductor uses OCI artifacts containing component descriptors (JSON files) for connectors and transformations. This enables the platform to: + - Discover available connectors and transformations - Render UI forms for configuration - Validate pipeline configurations @@ -78,6 +75,7 @@ The conductor uses OCI artifacts containing component descriptors (JSON files) f ### How it Works At conductor startup, the OCI artifact is downloaded using **ORAS** (OCI Registry as Storage): + - Downloads the descriptor OCI artifact to local filesystem - Extracts all descriptor JSON files - Adds ~5-15 seconds to startup time @@ -105,7 +103,7 @@ conductor: descriptors: official: enabled: true - tag: 3.5.0 # Override just the tag + tag: 3.5.0 # Override just the tag ``` To disable descriptors: @@ -123,7 +121,7 @@ Descriptor OCI artifacts should follow this structure: ``` debezium-descriptors:nightly -└── / +└── / ├── manifest.json ├── source-connector/ │ └── io.debezium.connector.mysql.MySqlConnector.json @@ -161,7 +159,7 @@ server: - **Default behavior**: Leave empty to let the operator decide ```yaml server: - image: "" # Operator's ServerImageProvider determines the image + image: "" # Operator's ServerImageProvider determines the image ``` # Install @@ -170,22 +168,22 @@ server: helm dependency build ``` -Thi will download the required [Debezium Operator](https://github.com/debezium/debezium-operator) chart. +This will download the required [Debezium Operator](https://github.com/debezium/debezium-operator) chart. ```shell -helm install . +helm install . ``` # Uninstall -Find the release name you want to uninstall +Find the release name you want to uninstall: ```shell helm list --all ``` -then uninstall it +then uninstall it: ```shell -helm uninstall +helm uninstall ``` \ No newline at end of file diff --git a/helm/templates/conductor-deployment.yaml b/helm/templates/conductor-deployment.yaml index 25aedd47..159b1346 100644 --- a/helm/templates/conductor-deployment.yaml +++ b/helm/templates/conductor-deployment.yaml @@ -22,7 +22,9 @@ spec: app.kubernetes.io/name: conductor spec: serviceAccountName: conductor-service-account -{{- $pullSecrets := coalesce .Values.conductor.imagePullSecrets .Values.imagePullSecrets | default list }} +{{- $conductorSecrets := .Values.conductor.imagePullSecrets | default list }} +{{- $globalSecrets := .Values.imagePullSecrets | default list }} +{{- $pullSecrets := ternary $conductorSecrets $globalSecrets (gt (len $conductorSecrets) 0) }} {{- if gt (len $pullSecrets) 0 }} imagePullSecrets: {{- toYaml $pullSecrets | nindent 8 }} @@ -90,4 +92,4 @@ spec: - name: {{ .name }} value: {{ .value }} {{- end }} - restartPolicy: Always \ No newline at end of file + restartPolicy: Always From 78beff6b1357de85604b092f0d0afec9e028e034 Mon Sep 17 00:00:00 2001 From: Bhawani Shanker Sharma Date: Fri, 15 May 2026 11:26:18 +0530 Subject: [PATCH 44/44] debezium/dbz#1851 Add imagePullSecrets and extend imagePullPolicy to all chart images Signed-off-by: Bhawani Shanker Sharma --- helm/README.md | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/helm/README.md b/helm/README.md index 7925b137..29523ff1 100644 --- a/helm/README.md +++ b/helm/README.md @@ -103,7 +103,7 @@ conductor: descriptors: official: enabled: true - tag: 3.5.0 # Override just the tag + tag: 3.5.0 ``` To disable descriptors: @@ -118,17 +118,13 @@ conductor: ### OCI Artifact Structure Descriptor OCI artifacts should follow this structure: - -``` debezium-descriptors:nightly └── / - ├── manifest.json - ├── source-connector/ - │ └── io.debezium.connector.mysql.MySqlConnector.json - └── transformation/ - └── io.debezium.transforms.ExtractNewRecordState.json -``` - +├── manifest.json +├── source-connector/ +│ └── io.debezium.connector.mysql.MySqlConnector.json +└── transformation/ +└── io.debezium.transforms.ExtractNewRecordState.json The artifact contents are extracted to the configured `mountPath`. ## Debezium Server Image Configuration @@ -145,22 +141,22 @@ server: ### Use Cases - **Pinning to a specific version**: Ensure all pipelines use a specific Debezium Server version - ```yaml +```yaml server: image: quay.io/debezium/server:3.0.0.Final - ``` +``` - **Using custom server image**: Deploy pipelines with a customized Debezium Server image - ```yaml +```yaml server: image: myregistry.io/custom-debezium-server:latest - ``` +``` - **Default behavior**: Leave empty to let the operator decide - ```yaml +```yaml server: - image: "" # Operator's ServerImageProvider determines the image - ``` + image: "" +``` # Install @@ -186,4 +182,4 @@ then uninstall it: ```shell helm uninstall -``` \ No newline at end of file +```