Skip to content

PoC: DF without SupportedDataFrameFormat #1858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Jolanrensen wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

PoC: DF without SupportedDataFrameFormat #1858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 0 additions & 80 deletions core/api/core.api
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5756,29 +5756,13 @@ public final class org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImp
public static final fun toStandaloneSnippet (Lorg/jetbrains/kotlinx/dataframe/codeGen/CodeWithTypeCastGenerator;Ljava/lang/String;Ljava/util/List;)Ljava/lang/String;
}

public abstract interface class org/jetbrains/kotlinx/dataframe/impl/codeGen/DfReadResult {
}

public final class org/jetbrains/kotlinx/dataframe/impl/codeGen/DfReadResult$Error : org/jetbrains/kotlinx/dataframe/impl/codeGen/DfReadResult {
public fun <init> (Ljava/lang/Throwable;)V
public final fun getReason ()Ljava/lang/Throwable;
}

public final class org/jetbrains/kotlinx/dataframe/impl/codeGen/DfReadResult$Success : org/jetbrains/kotlinx/dataframe/impl/codeGen/DfReadResult {
public fun <init> (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lorg/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat;)V
public final fun getFormat ()Lorg/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat;
public final fun getReadDfMethod (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/codeGen/DefaultReadDfMethod;
public final fun getSchema ()Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
}

public final class org/jetbrains/kotlinx/dataframe/impl/codeGen/NameNormalizerImplKt {
public static final fun from (Lorg/jetbrains/kotlinx/dataframe/codeGen/NameNormalizer$Companion;Ljava/util/Set;)Lorg/jetbrains/kotlinx/dataframe/codeGen/NameNormalizer;
public static final fun id (Lorg/jetbrains/kotlinx/dataframe/codeGen/NameNormalizer$Companion;)Lorg/jetbrains/kotlinx/dataframe/codeGen/NameNormalizer;
}

public final class org/jetbrains/kotlinx/dataframe/impl/codeGen/SchemaReaderKt {
public static final fun getUrlCodeGenReader (Lorg/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator$Companion;)Lkotlin/jvm/functions/Function4;
public static final fun getUrlDfReader (Lorg/jetbrains/kotlinx/dataframe/codeGen/CodeGenerator$Companion;)Lkotlin/jvm/functions/Function2;
}

public abstract interface class org/jetbrains/kotlinx/dataframe/impl/columns/ColumnsList : org/jetbrains/kotlinx/dataframe/columns/ColumnSet {
Expand Down Expand Up @@ -5847,19 +5831,6 @@ public final class org/jetbrains/kotlinx/dataframe/impl/schema/UtilsKt {
public static final fun getSchema (Lkotlin/reflect/KClass;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
}

public final class org/jetbrains/kotlinx/dataframe/io/CSV : org/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat {
public fun <init> ()V
public fun <init> (C)V
public synthetic fun <init> (CILkotlin/jvm/internal/DefaultConstructorMarker;)V
public fun acceptsExtension (Ljava/lang/String;)Z
public fun acceptsSample (Lorg/jetbrains/kotlinx/dataframe/io/SupportedFormatSample;)Z
public fun createDefaultReadMethod (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/codeGen/DefaultReadDfMethod;
public fun getTestOrder ()I
public fun readDataFrame (Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public fun readDataFrame (Ljava/io/InputStream;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public fun readDataFrame (Ljava/nio/file/Path;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
}

public final class org/jetbrains/kotlinx/dataframe/io/CSVType : java/lang/Enum {
public static final field DEFAULT Lorg/jetbrains/kotlinx/dataframe/io/CSVType;
public static final field TDF Lorg/jetbrains/kotlinx/dataframe/io/CSVType;
Expand Down Expand Up @@ -6078,37 +6049,6 @@ public final class org/jetbrains/kotlinx/dataframe/io/DisplayConfiguration$Compa
public final fun getDEFAULT ()Lorg/jetbrains/kotlinx/dataframe/io/DisplayConfiguration;
}

public final class org/jetbrains/kotlinx/dataframe/io/GuessKt {
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/lang/String;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/net/URL;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/nio/file/Path;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/lang/String;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/net/URL;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/nio/file/Path;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/io/File;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/lang/String;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/net/URL;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/nio/file/Path;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/io/File;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/lang/String;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/net/URL;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static synthetic fun read$default (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/nio/file/Path;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static final fun readDataFrame (Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun readDataFrame (Ljava/net/URL;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun readDataFrame (Ljava/nio/file/Path;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun readDataFrame$default (Ljava/io/File;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun readDataFrame$default (Ljava/net/URL;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun readDataFrame$default (Ljava/nio/file/Path;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun readDataRow (Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static final fun readDataRow (Ljava/net/URL;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static final fun readDataRow (Ljava/nio/file/Path;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static synthetic fun readDataRow$default (Ljava/io/File;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static synthetic fun readDataRow$default (Ljava/net/URL;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
public static synthetic fun readDataRow$default (Ljava/nio/file/Path;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
}

public final class org/jetbrains/kotlinx/dataframe/io/HtmlKt {
public static final fun toHTML (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lorg/jetbrains/kotlinx/dataframe/io/DisplayConfiguration;Lorg/jetbrains/kotlinx/dataframe/jupyter/CellRenderer;Lkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/io/DataFrameHtmlData;
public static synthetic fun toHTML$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lorg/jetbrains/kotlinx/dataframe/io/DisplayConfiguration;Lorg/jetbrains/kotlinx/dataframe/jupyter/CellRenderer;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/DataFrameHtmlData;
Expand Down Expand Up @@ -6171,15 +6111,6 @@ public abstract interface class org/jetbrains/kotlinx/dataframe/io/SupportedCode
public static synthetic fun readCodeForGeneration$default (Lorg/jetbrains/kotlinx/dataframe/io/SupportedCodeGenerationFormat;Ljava/io/InputStream;Ljava/lang/String;ZILjava/lang/Object;)Ljava/lang/String;
}

public abstract interface class org/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat : org/jetbrains/kotlinx/dataframe/io/SupportedFormat {
public fun readDataFrame (Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public abstract fun readDataFrame (Ljava/io/InputStream;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public abstract fun readDataFrame (Ljava/nio/file/Path;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun readDataFrame$default (Lorg/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat;Ljava/io/File;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun readDataFrame$default (Lorg/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat;Ljava/io/InputStream;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static synthetic fun readDataFrame$default (Lorg/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat;Ljava/nio/file/Path;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
}

public abstract interface class org/jetbrains/kotlinx/dataframe/io/SupportedFormat {
public abstract fun acceptsExtension (Ljava/lang/String;)Z
public abstract fun acceptsSample (Lorg/jetbrains/kotlinx/dataframe/io/SupportedFormatSample;)Z
Expand Down Expand Up @@ -6261,17 +6192,6 @@ public final class org/jetbrains/kotlinx/dataframe/io/SupportedFormatSample$Path
public final synthetic fun unbox-impl ()Ljava/lang/String;
}

public final class org/jetbrains/kotlinx/dataframe/io/TSV : org/jetbrains/kotlinx/dataframe/io/SupportedDataFrameFormat {
public fun <init> ()V
public fun acceptsExtension (Ljava/lang/String;)Z
public fun acceptsSample (Lorg/jetbrains/kotlinx/dataframe/io/SupportedFormatSample;)Z
public fun createDefaultReadMethod (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/codeGen/DefaultReadDfMethod;
public fun getTestOrder ()I
public fun readDataFrame (Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public fun readDataFrame (Ljava/io/InputStream;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public fun readDataFrame (Ljava/nio/file/Path;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
}

public final class org/jetbrains/kotlinx/dataframe/io/TsvKt {
public static final fun readTSV (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/io/File;Ljava/util/List;Ljava/util/Map;ILjava/lang/Integer;ZLjava/nio/charset/Charset;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
public static final fun readTSV (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/io/InputStream;Ljava/util/List;ZLjava/util/Map;ILjava/lang/Integer;ZLjava/nio/charset/Charset;Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
Expand Down
8 changes: 0 additions & 8 deletions core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/DefaultReadDfMethods.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,6 @@ public interface DefaultReadDfMethod {
// Used APIs
private const val CAST = "cast"
private const val VERIFY = "verify" // cast(true) is obscure, i think it's better to use named argument here
private const val READ_CSV = "readCSV"
private const val READ_TSV = "readTSV"
private const val READ_JDBC = "readJdbc"

public abstract class AbstractDefaultReadMethod(
private val path: String?,
Expand Down Expand Up @@ -80,8 +77,3 @@ public abstract class AbstractDefaultReadMethod(

override val additionalImports: List<String> = listOf("import org.jetbrains.kotlinx.dataframe.io.$methodName")
}

internal class DefaultReadCsvMethod(path: String?, arguments: MethodArguments) :
AbstractDefaultReadMethod(path, arguments, READ_CSV)

internal class DefaultReadTsvMethod(path: String?) : AbstractDefaultReadMethod(path, MethodArguments.EMPTY, READ_TSV)
41 changes: 1 addition & 40 deletions core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/SchemaReader.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,52 +1,13 @@
package org.jetbrains.kotlinx.dataframe.impl.codeGen

import org.jetbrains.kotlinx.dataframe.AnyFrame
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.dataframe.io.SupportedCodeGenerationFormat
import org.jetbrains.kotlinx.dataframe.io.SupportedDataFrameFormat
import org.jetbrains.kotlinx.dataframe.io.SupportedFormat
import org.jetbrains.kotlinx.dataframe.io.guessFormat
import org.jetbrains.kotlinx.dataframe.io.read
import org.jetbrains.kotlinx.dataframe.io.readCodeForGeneration
import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
import java.net.URL

/**
* Reader that can read a dataframe from a URL. It tries to guess the format based on the given [formats] and returns
* [DfReadResult.Success], or returns [DfReadResult.Error] if it fails.
*/
public val CodeGenerator.Companion.urlDfReader: (url: URL, formats: List<SupportedFormat>) -> DfReadResult
get() = { url, formats ->
try {
val (format, df) = url.openStream().use {
DataFrame.read(
stream = it,
format = guessFormat(url, formats) as? SupportedDataFrameFormat?,
formats = formats.filterIsInstance<SupportedDataFrameFormat>(),
)
}
DfReadResult.Success(df, format)
} catch (e: Throwable) {
DfReadResult.Error(e)
}
}

public sealed interface DfReadResult {

public class Success(private val df: AnyFrame, public val format: SupportedDataFrameFormat) : DfReadResult {
public fun getReadDfMethod(pathRepresentation: String?): DefaultReadDfMethod =
format.createDefaultReadMethod(pathRepresentation)

public val schema: DataFrameSchema = df.schema()
}

public class Error(public val reason: Throwable) : DfReadResult
}

/**
* Reader that can read data from a URL and generate code (type schema representations) for it.
* It tries to guess the format based on the given [formats] and returns [CodeGenerationReadResult.Success],
Expand All @@ -55,7 +16,7 @@ public sealed interface DfReadResult {
public val CodeGenerator.Companion.urlCodeGenReader: (
url: URL,
name: String,
formats: List<SupportedFormat>,
formats: List<SupportedCodeGenerationFormat>,
generateHelperCompanionObject: Boolean,
) -> CodeGenerationReadResult
get() = { url, name, formats, generateHelperCompanionObject ->
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/common.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public fun catchHttpResponse(url: URL, body: (InputStream) -> AnyFrame): AnyFram
val response = connection.responseMessage
try {
// attempt to read error response as dataframe
return DataFrame.read(connection.errorStream).df
return body(connection.errorStream)
} catch (_: Exception) {
throw RuntimeException("Server returned HTTP response code: $code. Response: $response")
}
Expand Down
29 changes: 1 addition & 28 deletions core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/csv.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.api.ParserOptions
import org.jetbrains.kotlinx.dataframe.api.forEach
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadCsvMethod
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.dataframe.impl.api.parse
import org.jetbrains.kotlinx.dataframe.impl.io.readDelimImpl
import org.jetbrains.kotlinx.dataframe.util.APACHE_CSV
Expand Down Expand Up @@ -57,38 +55,13 @@ import java.math.BigDecimal
import java.math.BigInteger
import java.net.URL
import java.nio.charset.Charset
import java.nio.file.Path
import java.util.zip.GZIPInputStream
import kotlin.reflect.KClass
import kotlin.reflect.KType
import kotlin.reflect.typeOf
import kotlin.time.Duration
import kotlin.time.Instant as StdlibInstant
import kotlinx.datetime.Instant as DeprecatedInstant

@Deprecated(message = APACHE_CSV, level = DeprecationLevel.WARNING)
public class CSV(private val delimiter: Char = ',') : SupportedDataFrameFormat {
override fun readDataFrame(stream: InputStream, header: List<String>): AnyFrame =
DataFrame.readCSV(stream = stream, delimiter = delimiter, header = header)

override fun readDataFrame(file: File, header: List<String>): AnyFrame =
DataFrame.readCSV(file = file, delimiter = delimiter, header = header)

override fun readDataFrame(path: Path, header: List<String>): AnyFrame =
// core CSV impl is deprecated, delegate via File to preserve module boundaries
DataFrame.readCSV(file = path.toFile(), delimiter = delimiter, header = header)

override fun acceptsExtension(ext: String): Boolean = ext == "csv"

override fun acceptsSample(sample: SupportedFormatSample): Boolean = true // Extension is enough

override val testOrder: Int = 20_001

override fun createDefaultReadMethod(pathRepresentation: String?): DefaultReadDfMethod {
val arguments = MethodArguments().add("delimiter", typeOf<Char>(), "'%L'", delimiter)
return DefaultReadCsvMethod(pathRepresentation, arguments)
}
}
import kotlin.time.Instant as StdlibInstant

@Deprecated(
message = APACHE_CSV,
Expand Down
Loading
Loading