Compare commits
161 Commits
v0.4.0-dev
...
master
Author | SHA1 | Date | |
---|---|---|---|
c754dc3471 | |||
8f7d754301 | |||
c923c3e7d3 | |||
99b2d941c8 | |||
191af77f57 | |||
79759c5256 | |||
2eb965e563 | |||
1b29e377ca | |||
2634a19285 | |||
261c415d3d | |||
e52d509c2b | |||
706521a6b6 | |||
94000689da | |||
851fdda311 | |||
cbbcd18df3 | |||
dc2bf5da83 | |||
f732b85cc5 | |||
259b882e63 | |||
526f230300 | |||
|
7fca5db390 | ||
|
be160ba98a | ||
|
182f206b88 | ||
3806f97c77 | |||
d5ebef404f | |||
3644533043 | |||
ee5afcdafe | |||
de476fb273 | |||
a136db16ff | |||
a699c36f8e | |||
2aba1b48dc | |||
cfa20eedba | |||
|
f78f0f814f | ||
4e7ead0763 | |||
4543648cda | |||
b6949310ea | |||
2c2f33427a | |||
29fa30fb51 | |||
f3afb5e9fe | |||
61c8df9eb0 | |||
707b59e6fc | |||
add400b324 | |||
58c5355e25 | |||
f83b759e75 | |||
7d88f828d7 | |||
5d7ddb4e00 | |||
82838b6a92 | |||
e41fdfc086 | |||
4117a05df4 | |||
5406a6a64c | |||
0cc4dc0db7 | |||
233639f0b6 | |||
70bd92f019 | |||
f8eea45ed0 | |||
e14c0a695e | |||
3c1fe23366 | |||
81e2ad06cc | |||
6ca76cff17 | |||
9a24e1e392 | |||
9cceb44a90 | |||
0b68c1edae | |||
b8869570ce | |||
4833128857 | |||
6bd8a7acbc | |||
f5d32ba511 | |||
0fc2198832 | |||
a546552540 | |||
fe92e8fccf | |||
7d9189e15c | |||
0622bacc4d | |||
bedab0dc86 | |||
f0820a3bed | |||
665f317e4e | |||
82d37f4b55 | |||
6d396368b7 | |||
77857289f0 | |||
eaa9d40d60 | |||
6b41163ed3 | |||
e5000171f1 | |||
3c6bc15716 | |||
11143e4ba1 | |||
91621864c2 | |||
1e97165328 | |||
|
be8e971436 | ||
|
9cc30b1f4e | ||
|
7414e60192 | ||
|
8c0bc05a9a | ||
c480cd8e4d | |||
64e0c554cc | |||
532e0c253b | |||
c423dc214e | |||
d178c4ff0d | |||
|
387ab8747e | ||
3f54eee578 | |||
aded38254e | |||
00d964eef3 | |||
b07d281a83 | |||
81cdd38c40 | |||
0ad6852e36 | |||
a71bb732da | |||
|
acfe9c2f74 | ||
ce8074c104 | |||
922a3b07ee | |||
81abbe28a9 | |||
da9d6e7639 | |||
66c708d9fb | |||
c1065c2885 | |||
e5f422f9ca | |||
c01bc36d41 | |||
be2daca25e | |||
b968d735ce | |||
67554a8c98 | |||
24187722e4 | |||
d3c129526d | |||
5632487dca | |||
e432b07201 | |||
28a6914747 | |||
90a92c4121 | |||
b404615145 | |||
7aec2f3547 | |||
3ba5a9076b | |||
8763d63e28 | |||
4a76063093 | |||
5fbbac465a | |||
b387b21554 | |||
c8bd3390cb | |||
a3479e74f7 | |||
9f5b010847 | |||
679175391a | |||
14455c2b2b | |||
a9cec666a3 | |||
d2ea1a975e | |||
73b3bbe7fc | |||
474597777c | |||
9d3c7149b7 | |||
bc9cd3b5a8 | |||
254163bdef | |||
|
3bdaf332cb | ||
|
6e769d1089 | ||
|
e62ff61814 | ||
1037c45c0d | |||
82b328f797 | |||
a6f1e54255 | |||
3888b2d9e7 | |||
0acb6ec448 | |||
|
b616e3ad6d | ||
|
d1381cc98c | ||
|
543023d2df | ||
|
d769b0d389 | ||
|
3fc698dd09 | ||
|
32b986fc47 | ||
3e8421187f | |||
5afe0523f1 | |||
a7ecbfb763 | |||
2352f1cff1 | |||
1a983665f8 | |||
53393e7958 | |||
874a253292 | |||
187094d942 | |||
23f1d4f7fd | |||
f1f5f7a70c | |||
9fbc482549 |
21
.github/workflows/build.yml
vendored
21
.github/workflows/build.yml
vendored
@ -1,6 +1,9 @@
|
||||
name: Gradle build
|
||||
|
||||
on: [ push ]
|
||||
on:
|
||||
push:
|
||||
branches: [ dev, master ]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@ -8,20 +11,22 @@ jobs:
|
||||
matrix:
|
||||
os: [ macOS-latest, windows-latest ]
|
||||
runs-on: ${{matrix.os}}
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- name: Checkout the repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: DeLaGuardo/setup-graalvm@4.0
|
||||
with:
|
||||
java-version: 11
|
||||
- name: Add msys to path
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: SETX PATH "%PATH%;C:\msys64\mingw64\bin"
|
||||
graalvm: 21.2.0
|
||||
java: java11
|
||||
arch: amd64
|
||||
- name: Cache gradle
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.gradle/caches
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
~/.gradle/wrapper
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-gradle-
|
||||
@ -33,4 +38,4 @@ jobs:
|
||||
restore-keys: |
|
||||
${{ runner.os }}-gradle-
|
||||
- name: Build
|
||||
run: ./gradlew build --no-daemon --stacktrace
|
||||
run: ./gradlew build --build-cache --no-daemon --stacktrace
|
||||
|
31
.github/workflows/pages.yml
vendored
Normal file
31
.github/workflows/pages.yml
vendored
Normal file
@ -0,0 +1,31 @@
|
||||
name: Dokka publication
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
release:
|
||||
types: [ created ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- uses: actions/checkout@v3.0.0
|
||||
- uses: actions/setup-java@v3.0.0
|
||||
with:
|
||||
java-version: 11
|
||||
distribution: liberica
|
||||
- name: Cache konan
|
||||
uses: actions/cache@v3.0.1
|
||||
with:
|
||||
path: ~/.konan
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-gradle-
|
||||
- uses: gradle/gradle-build-action@v2.4.2
|
||||
with:
|
||||
arguments: dokkaHtmlMultiModule --no-parallel
|
||||
- uses: JamesIves/github-pages-deploy-action@v4.3.0
|
||||
with:
|
||||
branch: gh-pages
|
||||
folder: build/dokka/htmlMultiModule
|
57
.github/workflows/publish.yml
vendored
57
.github/workflows/publish.yml
vendored
@ -3,8 +3,7 @@ name: Gradle publish
|
||||
on:
|
||||
workflow_dispatch:
|
||||
release:
|
||||
types:
|
||||
- created
|
||||
types: [ created ]
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
@ -12,27 +11,16 @@ jobs:
|
||||
name: publish
|
||||
strategy:
|
||||
matrix:
|
||||
os: [macOS-latest, windows-latest]
|
||||
os: [ macOS-latest, windows-latest ]
|
||||
runs-on: ${{matrix.os}}
|
||||
steps:
|
||||
- name: Checkout the repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
- uses: actions/checkout@v3.0.0
|
||||
- uses: actions/setup-java@v3.10.0
|
||||
with:
|
||||
java-version: 11
|
||||
- name: Add msys to path
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: SETX PATH "%PATH%;C:\msys64\mingw64\bin"
|
||||
- name: Cache gradle
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.gradle/caches
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-gradle-
|
||||
distribution: liberica
|
||||
- name: Cache konan
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v3.0.1
|
||||
with:
|
||||
path: ~/.konan
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
@ -40,20 +28,23 @@ jobs:
|
||||
${{ runner.os }}-gradle-
|
||||
- name: Publish Windows Artifacts
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: >
|
||||
./gradlew release --no-daemon
|
||||
-Ppublishing.enabled=true
|
||||
-Ppublishing.github.user=${{ secrets.PUBLISHING_GITHUB_USER }}
|
||||
-Ppublishing.github.token=${{ secrets.PUBLISHING_GITHUB_TOKEN }}
|
||||
-Ppublishing.space.user=${{ secrets.PUBLISHING_SPACE_USER }}
|
||||
-Ppublishing.space.token=${{ secrets.PUBLISHING_SPACE_TOKEN }}
|
||||
uses: gradle/gradle-build-action@v2.4.2
|
||||
with:
|
||||
arguments: |
|
||||
publishAllPublicationsToSpaceRepository
|
||||
-Ppublishing.targets=all
|
||||
-Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
|
||||
-Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}
|
||||
- name: Publish Mac Artifacts
|
||||
if: matrix.os == 'macOS-latest'
|
||||
run: >
|
||||
./gradlew release --no-daemon
|
||||
-Ppublishing.enabled=true
|
||||
-Ppublishing.platform=macosX64
|
||||
-Ppublishing.github.user=${{ secrets.PUBLISHING_GITHUB_USER }}
|
||||
-Ppublishing.github.token=${{ secrets.PUBLISHING_GITHUB_TOKEN }}
|
||||
-Ppublishing.space.user=${{ secrets.PUBLISHING_SPACE_USER }}
|
||||
-Ppublishing.space.token=${{ secrets.PUBLISHING_SPACE_TOKEN }}
|
||||
uses: gradle/gradle-build-action@v2.4.2
|
||||
with:
|
||||
arguments: |
|
||||
publishMacosX64PublicationToSpaceRepository
|
||||
publishMacosArm64PublicationToSpaceRepository
|
||||
publishIosX64PublicationToSpaceRepository
|
||||
publishIosArm64PublicationToSpaceRepository
|
||||
publishIosSimulatorArm64PublicationToSpaceRepository
|
||||
-Ppublishing.targets=all
|
||||
-Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
|
||||
-Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}
|
||||
|
181
CHANGELOG.md
181
CHANGELOG.md
@ -1,26 +1,173 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
## Unreleased
|
||||
|
||||
### Added
|
||||
- LogManager plugin
|
||||
- dataforge-context API dependency on SLF4j
|
||||
|
||||
### Changed
|
||||
- Kotlin-logging moved from common to JVM and JS. Replaced by console for native.
|
||||
- Package changed to `space.kscience`
|
||||
|
||||
### Deprecated
|
||||
|
||||
### Removed
|
||||
- Common dependency on Kotlin-logging
|
||||
- Kotlinx-io fork dependency. Replaced by Ktor-io.
|
||||
|
||||
### Fixed
|
||||
|
||||
### Security
|
||||
|
||||
## [0.3.0]
|
||||
## 0.7.0 - 2023-11-26
|
||||
|
||||
### Added
|
||||
|
||||
- Obligatory `type: KType` and `descriptor` property for `MetaConverters`
|
||||
- Added separate `Meta`, `SealedMeta` and `ObservableMutableMeta` builders.
|
||||
|
||||
### Changed
|
||||
|
||||
- Meta converter `metaToObject` returns a non-nullable type. Additional method `metaToObjectOrNull` for nullable return.
|
||||
- Kotlin 1.9.20.
|
||||
- Migrated from ktor-io to kotlinx-io.
|
||||
- `MutableMeta` builder now returns a simplified version of meta that does not hold listeners.
|
||||
- More concise names for read/write methods in IO.
|
||||
- Remove unnecessary confusion with `get`/`getMeta` by removing `getMeta` from the interface.
|
||||
|
||||
### Deprecated
|
||||
|
||||
- `String.parseValue` is replaced with `Value.parse`
|
||||
|
||||
### Fixed
|
||||
|
||||
- Memory leak in SealedMeta builder
|
||||
|
||||
## 0.6.2 - 2023-07-29
|
||||
|
||||
### Changed
|
||||
|
||||
- Meta to Json serializer now serializes a single item with index as an array. It is important for plotly integration.
|
||||
- Meta to Json serializes Meta without children a value as literal or array instead of an object with `@value` field.
|
||||
|
||||
## 0.6.1 - 2023-03-31
|
||||
|
||||
### Added
|
||||
|
||||
- File cache for workspace
|
||||
- Smart task metadata transformation for workspace
|
||||
- Add `readOnly` property to descriptors
|
||||
- Add `specOrNull` delegate to meta and Scheme
|
||||
- Suspended read methods to the `Binary`
|
||||
- Synchronously accessed `meta` to all `DataSet`s
|
||||
- More fine-grained types in Action builders.
|
||||
|
||||
### Changed
|
||||
|
||||
- `Name::replaceLast` API
|
||||
- `PluginFactory` no longer requires plugin class
|
||||
- Collection<Named> toMap -> associateByName
|
||||
- Simplified `DFTL` envelope format. Closing symbols are unnecessary. Properties are discontinued.
|
||||
- Meta `get` method allows nullable receiver
|
||||
- `withDefault` functions do not add new keys to meta children and are consistent.
|
||||
- `dataforge.meta.values` package is merged into `dataforge.meta` for better star imports
|
||||
- Kotlin 1.8.20
|
||||
- `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension.
|
||||
- KTor 2.0
|
||||
- DataTree `items` call is blocking.
|
||||
- DataSet `getData` is no longer suspended and renamed to `get`
|
||||
- DataSet operates with sequences of data instead of flows
|
||||
- PartialEnvelope uses `Int` instead `UInt`.
|
||||
- `ActiveDataSet` renamed to `DataSource`
|
||||
- `selectOne`->`getByType`
|
||||
- Data traversal in `DataSet` is done via iterator
|
||||
- Remove all unnecessary properties for `IOFormat`
|
||||
- Separate interfaces for `IOReader` and `IOWriter`
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Context.fetch -> Context.request
|
||||
|
||||
### Fixed
|
||||
|
||||
- `readDataDirectory` does not split names with dots
|
||||
- Front matter reader does not crash on non-UTF files
|
||||
- Meta file name in readMeta from directory
|
||||
- Tagless and FrontMatter envelope partial readers fix.
|
||||
|
||||
## 0.5.2
|
||||
|
||||
### Added
|
||||
|
||||
- Yaml plugin
|
||||
- Partial fix to #53
|
||||
|
||||
### Fixed
|
||||
|
||||
- MutableMetaImpl attachment and checks
|
||||
- Listeners in observable meta are replaced by lists
|
||||
- JS number comparison bug.
|
||||
|
||||
## 0.5.0
|
||||
|
||||
### Added
|
||||
|
||||
- Experimental `listOfSpec` delegate.
|
||||
|
||||
### Changed
|
||||
|
||||
- **API breaking** Config is deprecated, use `ObservableMeta` instead.
|
||||
- **API breaking** Descriptor no has a member property `defaultValue` instead of `defaultItem()` extension. It caches default value state on the first call. It is done because computing default on each call is too expensive.
|
||||
- Kotlin 1.5.10
|
||||
- Build tools 0.10.0
|
||||
- Relaxed type restriction on `MetaConverter`. Now nullables are available.
|
||||
- **Huge API-breaking refactoring of Meta**. Meta now can have both value and children. There is only one kind of descriptor now.
|
||||
- **API breaking** `String.toName()` is replaced by `Name.parse()`
|
||||
- **API breaking** Configurable`config` changed to `meta`
|
||||
|
||||
### Removed
|
||||
|
||||
- `Config`
|
||||
- Public PluginManager mutability
|
||||
- Tables and tables-exposed moved to the separate project `tables.kt`
|
||||
- BinaryMetaFormat. Use CBOR encoding instead
|
||||
|
||||
### Fixed
|
||||
|
||||
- Proper json array index treatment.
|
||||
- Proper json index for single-value array.
|
||||
|
||||
## 0.4.0
|
||||
|
||||
### Added
|
||||
|
||||
- LogManager plugin
|
||||
- dataforge-context API dependency on SLF4j
|
||||
- Context `withEnv` and `fetch` methods to manipulate plugins without changing plugins after creation.
|
||||
- Split `ItemDescriptor` into builder and read-only part
|
||||
|
||||
### Changed
|
||||
|
||||
- Kotlin-logging moved from common to JVM and JS. Replaced by console for native.
|
||||
- Package changed to `space.kscience`
|
||||
- Scheme made observable
|
||||
- Global context is a variable (the singleton is hidden and will be deprecated in future)
|
||||
- Kotlin 1.5
|
||||
- Added blank builders for children context.
|
||||
- Refactor loggers
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Direct use of PluginManager
|
||||
|
||||
### Removed
|
||||
|
||||
- Common dependency on Kotlin-logging
|
||||
- Kotlinx-io fork dependency. Replaced by Ktor-io.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Scheme properties properly handle children property change.
|
||||
|
||||
## 0.3.0
|
||||
|
||||
### Added
|
||||
|
||||
- Yaml meta format based on yaml.kt
|
||||
- `Path` builders
|
||||
- Special ValueType for lists
|
||||
@ -28,6 +175,7 @@
|
||||
- Multiplatform yaml meta
|
||||
|
||||
### Changed
|
||||
|
||||
- `ListValue` and `DoubleArrayValue` implement `Iterable`.
|
||||
- Changed the logic of `Value::isList` to check for type instead of size
|
||||
- `Meta{}` builder made inline
|
||||
@ -41,18 +189,10 @@
|
||||
- \[Major breaking change\] Full refactor of DataTree/DataSource
|
||||
- \[Major Breaking change\] Replace KClass with KType in data. Remove direct access to constructors with types.
|
||||
|
||||
### Deprecated
|
||||
|
||||
### Removed
|
||||
|
||||
### Fixed
|
||||
|
||||
### Security
|
||||
|
||||
## [0.2.0]
|
||||
### Added
|
||||
## 0.2.0
|
||||
|
||||
### Changed
|
||||
|
||||
- Context content resolution refactor
|
||||
- Kotlin 1.4.10 (build tools 0.6.0)
|
||||
- Empty query in Name is null instead of ""
|
||||
@ -62,15 +202,16 @@
|
||||
- Configurable is no longer MutableItemProvider. All functionality moved to Scheme.
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Context activation API
|
||||
- TextRenderer
|
||||
|
||||
### Removed
|
||||
|
||||
- Functional server prototype
|
||||
- `dataforge-output` module
|
||||
|
||||
### Fixed
|
||||
|
||||
- Global context CoroutineScope resolution
|
||||
- Library mode compliance
|
||||
|
||||
### Security
|
||||
|
201
LICENSE
Normal file
201
LICENSE
Normal file
@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
39
README.md
39
README.md
@ -3,53 +3,36 @@
|
||||
|
||||
![Gradle build](https://github.com/mipt-npm/dataforge-core/workflows/Gradle%20build/badge.svg)
|
||||
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-context](dataforge-context)
|
||||
>
|
||||
### [dataforge-context](dataforge-context)
|
||||
> Context and provider definitions
|
||||
>
|
||||
> **Maturity**: DEVELOPMENT
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-data](dataforge-data)
|
||||
>
|
||||
### [dataforge-data](dataforge-data)
|
||||
>
|
||||
> **Maturity**: EXPERIMENTAL
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-io](dataforge-io)
|
||||
>
|
||||
### [dataforge-io](dataforge-io)
|
||||
> IO module
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
> **Maturity**: EXPERIMENTAL
|
||||
|
||||
* ### [dataforge-meta](dataforge-meta)
|
||||
>
|
||||
### [dataforge-meta](dataforge-meta)
|
||||
> Meta definition and basic operations on meta
|
||||
>
|
||||
> **Maturity**: DEVELOPMENT
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-scripting](dataforge-scripting)
|
||||
>
|
||||
### [dataforge-scripting](dataforge-scripting)
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-tables](dataforge-tables)
|
||||
>
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-workspace](dataforge-workspace)
|
||||
>
|
||||
### [dataforge-workspace](dataforge-workspace)
|
||||
>
|
||||
> **Maturity**: EXPERIMENTAL
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-io-yaml](dataforge-io/dataforge-io-yaml)
|
||||
### [dataforge-io/dataforge-io-yaml](dataforge-io/dataforge-io-yaml)
|
||||
> YAML meta converters and Front Matter envelope format
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
|
||||
|
@ -1,17 +1,23 @@
|
||||
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
||||
import space.kscience.gradle.useApache2Licence
|
||||
import space.kscience.gradle.useSPCTeam
|
||||
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.project")
|
||||
id("space.kscience.gradle.project")
|
||||
}
|
||||
|
||||
allprojects {
|
||||
group = "space.kscience"
|
||||
version = "0.4.0-dev-2"
|
||||
version = "0.7.0"
|
||||
}
|
||||
|
||||
subprojects {
|
||||
apply(plugin = "maven-publish")
|
||||
repositories {
|
||||
maven("https://dl.bintray.com/mipt-npm/kscience")
|
||||
maven("https://dl.bintray.com/mipt-npm/dev")
|
||||
|
||||
tasks.withType<KotlinCompile> {
|
||||
kotlinOptions {
|
||||
freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,8 +26,11 @@ readme {
|
||||
}
|
||||
|
||||
ksciencePublish {
|
||||
github("dataforge-core")
|
||||
space("https://maven.pkg.jetbrains.space/mipt-npm/p/sci/maven")
|
||||
pom("https://github.com/SciProgCentre/kmath") {
|
||||
useApache2Licence()
|
||||
useSPCTeam()
|
||||
}
|
||||
repository("spc","https://maven.sciprog.center/kscience")
|
||||
sonatype()
|
||||
}
|
||||
|
||||
|
23
dataforge-context/README.md
Normal file
23
dataforge-context/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-context
|
||||
|
||||
Context and provider definitions
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-context:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-context:0.7.0")
|
||||
}
|
||||
```
|
@ -3,21 +3,17 @@ public abstract class space/kscience/dataforge/context/AbstractPlugin : space/ks
|
||||
public fun <init> (Lspace/kscience/dataforge/meta/Meta;)V
|
||||
public synthetic fun <init> (Lspace/kscience/dataforge/meta/Meta;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
|
||||
public fun attach (Lspace/kscience/dataforge/context/Context;)V
|
||||
public fun content (Ljava/lang/String;)Ljava/util/Map;
|
||||
public final fun dependsOn ()Ljava/util/Map;
|
||||
public fun dependsOn ()Ljava/util/Map;
|
||||
public fun detach ()V
|
||||
public fun getContext ()Lspace/kscience/dataforge/context/Context;
|
||||
public fun getDefaultChainTarget ()Ljava/lang/String;
|
||||
public fun getDefaultTarget ()Ljava/lang/String;
|
||||
public fun getMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
public fun getName ()Lspace/kscience/dataforge/names/Name;
|
||||
protected final fun require (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)Lkotlin/properties/ReadOnlyProperty;
|
||||
public static synthetic fun require$default (Lspace/kscience/dataforge/context/AbstractPlugin;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty;
|
||||
public fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
public fun isAttached ()Z
|
||||
protected final fun require (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/meta/Meta;)Lkotlin/properties/ReadOnlyProperty;
|
||||
public static synthetic fun require$default (Lspace/kscience/dataforge/context/AbstractPlugin;Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/AbstractPluginKt {
|
||||
public static final fun toMap (Ljava/util/Collection;)Ljava/util/Map;
|
||||
public static final fun associateByName (Ljava/util/Collection;)Ljava/util/Map;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/ClassLoaderPlugin : space/kscience/dataforge/context/AbstractPlugin {
|
||||
@ -38,15 +34,17 @@ public final class space/kscience/dataforge/context/ClassLoaderPluginKt {
|
||||
public class space/kscience/dataforge/context/Context : kotlinx/coroutines/CoroutineScope, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/misc/Named, space/kscience/dataforge/provider/Provider {
|
||||
public static final field Companion Lspace/kscience/dataforge/context/Context$Companion;
|
||||
public static final field PROPERTY_TARGET Ljava/lang/String;
|
||||
public final fun buildContext (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
|
||||
public static synthetic fun buildContext$default (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
|
||||
public fun close ()V
|
||||
public fun content (Ljava/lang/String;)Ljava/util/Map;
|
||||
public final fun content (Ljava/lang/String;Z)Ljava/util/Map;
|
||||
public fun getCoroutineContext ()Lkotlin/coroutines/CoroutineContext;
|
||||
public fun getDefaultChainTarget ()Ljava/lang/String;
|
||||
public fun getDefaultTarget ()Ljava/lang/String;
|
||||
public final fun getName ()Lspace/kscience/dataforge/names/Name;
|
||||
public final fun getParent ()Lspace/kscience/dataforge/context/Context;
|
||||
public final fun getPlugins ()Lspace/kscience/dataforge/context/PluginManager;
|
||||
public final fun getProperties ()Lspace/kscience/dataforge/meta/Laminate;
|
||||
public fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
}
|
||||
|
||||
@ -58,51 +56,62 @@ public abstract interface class space/kscience/dataforge/context/ContextAware {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/ContextBuilder {
|
||||
public fun <init> ()V
|
||||
public fun <init> (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;)V
|
||||
public synthetic fun <init> (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
|
||||
public final fun build ()Lspace/kscience/dataforge/context/Context;
|
||||
public final fun getName ()Ljava/lang/String;
|
||||
public final fun getName ()Lspace/kscience/dataforge/names/Name;
|
||||
public final fun plugin (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)V
|
||||
public final fun plugin (Lspace/kscience/dataforge/context/Plugin;)V
|
||||
public final fun plugin (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;)V
|
||||
public final fun plugin (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)V
|
||||
public final fun plugin (Lspace/kscience/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;)V
|
||||
public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
|
||||
public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
|
||||
public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
|
||||
public final fun properties (Lkotlin/jvm/functions/Function1;)V
|
||||
public final fun setName (Ljava/lang/String;)V
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/ContextKt {
|
||||
public static final fun Context (Ljava/lang/String;Lspace/kscience/dataforge/context/Context;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
|
||||
public static synthetic fun Context$default (Ljava/lang/String;Lspace/kscience/dataforge/context/Context;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
|
||||
public final class space/kscience/dataforge/context/ContextBuilderKt {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/DefaultLogManager : space/kscience/dataforge/context/AbstractPlugin, space/kscience/dataforge/context/LogManager {
|
||||
public static final field Companion Lspace/kscience/dataforge/context/DefaultLogManager$Companion;
|
||||
public fun <init> ()V
|
||||
public fun getDefaultLogger ()Lspace/kscience/dataforge/context/Logger;
|
||||
public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public fun logger (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/context/Logger;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/DefaultLogManager$Companion : space/kscience/dataforge/context/PluginFactory {
|
||||
public synthetic fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
|
||||
public fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/context/DefaultLogManager;
|
||||
public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/context/Factory {
|
||||
public abstract fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
|
||||
public abstract fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/Factory$DefaultImpls {
|
||||
public final class space/kscience/dataforge/context/FactoryKt {
|
||||
public static final fun invoke (Lspace/kscience/dataforge/context/Factory;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
|
||||
public static synthetic fun invoke$default (Lspace/kscience/dataforge/context/Factory;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;ILjava/lang/Object;)Ljava/lang/Object;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/Global : space/kscience/dataforge/context/Context {
|
||||
public static final field INSTANCE Lspace/kscience/dataforge/context/Global;
|
||||
public fun close ()V
|
||||
public final fun context (Ljava/lang/String;Lspace/kscience/dataforge/context/Context;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
|
||||
public static synthetic fun context$default (Lspace/kscience/dataforge/context/Global;Ljava/lang/String;Lspace/kscience/dataforge/context/Context;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
|
||||
public final fun getContext (Ljava/lang/String;)Lspace/kscience/dataforge/context/Context;
|
||||
public fun getCoroutineContext ()Lkotlin/coroutines/CoroutineContext;
|
||||
public final fun getLogger ()Lspace/kscience/dataforge/context/LogManager;
|
||||
public final class space/kscience/dataforge/context/GlobalKt {
|
||||
public static final fun Context (Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
|
||||
public static synthetic fun Context$default (Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
|
||||
public static final fun getGlobal ()Lspace/kscience/dataforge/context/Context;
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/context/LogManager : space/kscience/dataforge/context/Logable, space/kscience/dataforge/context/Plugin {
|
||||
public abstract interface class space/kscience/dataforge/context/LogManager : space/kscience/dataforge/context/Logger, space/kscience/dataforge/context/Plugin {
|
||||
public static final field Companion Lspace/kscience/dataforge/context/LogManager$Companion;
|
||||
public static final field DEBUG Ljava/lang/String;
|
||||
public static final field ERROR Ljava/lang/String;
|
||||
public static final field INFO Ljava/lang/String;
|
||||
public static final field TRACE Ljava/lang/String;
|
||||
public static final field WARNING Ljava/lang/String;
|
||||
public abstract fun getDefaultLogger ()Lspace/kscience/dataforge/context/Logger;
|
||||
public fun log (Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
|
||||
public fun log (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
|
||||
public abstract fun logger (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/context/Logger;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/LogManager$Companion {
|
||||
@ -113,33 +122,19 @@ public final class space/kscience/dataforge/context/LogManager$Companion {
|
||||
public static final field WARNING Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/LogManager$DefaultImpls {
|
||||
public static fun content (Lspace/kscience/dataforge/context/LogManager;Ljava/lang/String;)Ljava/util/Map;
|
||||
public static fun getDefaultChainTarget (Lspace/kscience/dataforge/context/LogManager;)Ljava/lang/String;
|
||||
public static fun getDefaultTarget (Lspace/kscience/dataforge/context/LogManager;)Ljava/lang/String;
|
||||
public static fun getName (Lspace/kscience/dataforge/context/LogManager;)Lspace/kscience/dataforge/names/Name;
|
||||
public static fun toMeta (Lspace/kscience/dataforge/context/LogManager;)Lspace/kscience/dataforge/meta/Meta;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/LogManagerKt {
|
||||
public static final fun debug (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)V
|
||||
public static synthetic fun debug$default (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)V
|
||||
public static final fun error (Lspace/kscience/dataforge/context/Logable;Ljava/lang/Throwable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)V
|
||||
public static final fun error (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)V
|
||||
public static synthetic fun error$default (Lspace/kscience/dataforge/context/Logable;Ljava/lang/Throwable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)V
|
||||
public static synthetic fun error$default (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)V
|
||||
public static final fun debug (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
|
||||
public static final fun error (Lspace/kscience/dataforge/context/Logger;Ljava/lang/Throwable;Lkotlin/jvm/functions/Function0;)V
|
||||
public static final fun error (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
|
||||
public static final fun getLogger (Lspace/kscience/dataforge/context/Context;)Lspace/kscience/dataforge/context/LogManager;
|
||||
public static final fun getLogger (Lspace/kscience/dataforge/context/ContextAware;)Lspace/kscience/dataforge/context/Logable;
|
||||
public static final fun info (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)V
|
||||
public static synthetic fun info$default (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)V
|
||||
public static final fun trace (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)V
|
||||
public static synthetic fun trace$default (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)V
|
||||
public static final fun warn (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;)V
|
||||
public static synthetic fun warn$default (Lspace/kscience/dataforge/context/Logable;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function0;ILjava/lang/Object;)V
|
||||
public static final fun getLogger (Lspace/kscience/dataforge/context/ContextAware;)Lspace/kscience/dataforge/context/Logger;
|
||||
public static final fun info (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
|
||||
public static final fun trace (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
|
||||
public static final fun warn (Lspace/kscience/dataforge/context/Logger;Lkotlin/jvm/functions/Function0;)V
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/context/Logable {
|
||||
public abstract fun log (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
|
||||
public abstract interface class space/kscience/dataforge/context/Logger {
|
||||
public abstract fun log (Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/context/Plugin : space/kscience/dataforge/context/ContextAware, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/misc/Named, space/kscience/dataforge/provider/Provider {
|
||||
@ -149,28 +144,36 @@ public abstract interface class space/kscience/dataforge/context/Plugin : space/
|
||||
public abstract fun dependsOn ()Ljava/util/Map;
|
||||
public abstract fun detach ()V
|
||||
public abstract fun getMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
public abstract fun getName ()Lspace/kscience/dataforge/names/Name;
|
||||
public fun getName ()Lspace/kscience/dataforge/names/Name;
|
||||
public abstract fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public abstract fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
public abstract fun isAttached ()Z
|
||||
public fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/Plugin$Companion {
|
||||
public static final field TARGET Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/Plugin$DefaultImpls {
|
||||
public static fun content (Lspace/kscience/dataforge/context/Plugin;Ljava/lang/String;)Ljava/util/Map;
|
||||
public static fun getDefaultChainTarget (Lspace/kscience/dataforge/context/Plugin;)Ljava/lang/String;
|
||||
public static fun getDefaultTarget (Lspace/kscience/dataforge/context/Plugin;)Ljava/lang/String;
|
||||
public static fun getName (Lspace/kscience/dataforge/context/Plugin;)Lspace/kscience/dataforge/names/Name;
|
||||
public static fun toMeta (Lspace/kscience/dataforge/context/Plugin;)Lspace/kscience/dataforge/meta/Meta;
|
||||
public final class space/kscience/dataforge/context/PluginBuilder {
|
||||
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
|
||||
public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
|
||||
public final fun build ()Lspace/kscience/dataforge/context/PluginFactory;
|
||||
public final fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public final fun provides (Ljava/lang/String;Ljava/util/Map;)V
|
||||
public final fun provides (Ljava/lang/String;[Lspace/kscience/dataforge/misc/Named;)V
|
||||
public final fun requires (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)V
|
||||
public static synthetic fun requires$default (Lspace/kscience/dataforge/context/PluginBuilder;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)V
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginBuilderKt {
|
||||
public static final fun PluginFactory (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/PluginFactory;
|
||||
public static synthetic fun PluginFactory$default (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/PluginFactory;
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/context/PluginFactory : space/kscience/dataforge/context/Factory {
|
||||
public static final field Companion Lspace/kscience/dataforge/context/PluginFactory$Companion;
|
||||
public static final field TYPE Ljava/lang/String;
|
||||
public abstract fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public abstract fun getType ()Lkotlin/reflect/KClass;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginFactory$Companion {
|
||||
@ -178,25 +181,15 @@ public final class space/kscience/dataforge/context/PluginFactory$Companion {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginManager : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker, space/kscience/dataforge/context/ContextAware {
|
||||
public fun <init> (Lspace/kscience/dataforge/context/Context;)V
|
||||
public final fun fetch (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;Z)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun fetch (Lspace/kscience/dataforge/context/PluginFactory;ZLkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun fetch$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ZILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun fetch$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginFactory;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun find (ZLkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun find$default (Lspace/kscience/dataforge/context/PluginManager;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun get (Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;Z)Ljava/lang/Object;
|
||||
public final fun get (Lspace/kscience/dataforge/context/PluginTag;Z)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun get$default (Lspace/kscience/dataforge/context/PluginManager;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Ljava/lang/Object;
|
||||
public static synthetic fun get$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun getByType (Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;Z)Ljava/lang/Object;
|
||||
public static synthetic fun getByType$default (Lspace/kscience/dataforge/context/PluginManager;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Ljava/lang/Object;
|
||||
public fun getContext ()Lspace/kscience/dataforge/context/Context;
|
||||
public fun iterator ()Ljava/util/Iterator;
|
||||
public final fun list (Z)Ljava/util/Collection;
|
||||
public final fun load (Lspace/kscience/dataforge/context/Plugin;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun load (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun load (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun load$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun remove (Lspace/kscience/dataforge/context/Plugin;)V
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginTag : space/kscience/dataforge/meta/MetaRepr {
|
||||
@ -219,14 +212,25 @@ public final class space/kscience/dataforge/context/PluginTag : space/kscience/d
|
||||
public fun toString ()Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginTag$$serializer : kotlinx/serialization/internal/GeneratedSerializer {
|
||||
public static final field INSTANCE Lspace/kscience/dataforge/context/PluginTag$$serializer;
|
||||
public fun childSerializers ()[Lkotlinx/serialization/KSerializer;
|
||||
public synthetic fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Ljava/lang/Object;
|
||||
public fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/context/PluginTag;
|
||||
public fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor;
|
||||
public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V
|
||||
public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/context/PluginTag;)V
|
||||
public fun typeParametersSerializers ()[Lkotlinx/serialization/KSerializer;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginTag$Companion {
|
||||
public final fun fromString (Ljava/lang/String;)Lspace/kscience/dataforge/context/PluginTag;
|
||||
public final fun serializer ()Lkotlinx/serialization/KSerializer;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/ResolveKt {
|
||||
public static final fun gather (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;Z)Ljava/util/Map;
|
||||
public static synthetic fun gather$default (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;ZILjava/lang/Object;)Ljava/util/Map;
|
||||
public static synthetic fun gatherInSequence$default (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;ZILjava/lang/Object;)Lkotlin/sequences/Sequence;
|
||||
public static final fun getValues (Lkotlin/sequences/Sequence;)Lkotlin/sequences/Sequence;
|
||||
public static final fun resolve (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;Lkotlin/reflect/KClass;)Ljava/lang/Object;
|
||||
}
|
||||
@ -234,49 +238,24 @@ public final class space/kscience/dataforge/context/ResolveKt {
|
||||
public final class space/kscience/dataforge/context/SlfLogManager : space/kscience/dataforge/context/AbstractPlugin, space/kscience/dataforge/context/LogManager {
|
||||
public static final field Companion Lspace/kscience/dataforge/context/SlfLogManager$Companion;
|
||||
public fun <init> ()V
|
||||
public fun getDefaultLogger ()Lspace/kscience/dataforge/context/Logger;
|
||||
public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public fun log (Lspace/kscience/dataforge/names/Name;Ljava/lang/String;Lkotlin/jvm/functions/Function0;)V
|
||||
public fun logger (Lspace/kscience/dataforge/names/Name;)Lspace/kscience/dataforge/context/Logger;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/SlfLogManager$Companion : space/kscience/dataforge/context/PluginFactory {
|
||||
public synthetic fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
|
||||
public fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/context/SlfLogManager;
|
||||
public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public fun getType ()Lkotlin/reflect/KClass;
|
||||
public synthetic fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
|
||||
public fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Lspace/kscience/dataforge/context/SlfLogManager;
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/Attribute : java/lang/annotation/Annotation {
|
||||
public abstract fun key ()Ljava/lang/String;
|
||||
public abstract fun value ()Ljava/lang/String;
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/Attributes : java/lang/annotation/Annotation {
|
||||
public abstract fun attrs ()[Lspace/kscience/dataforge/descriptors/Attribute;
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/ItemDef : java/lang/annotation/Annotation {
|
||||
public abstract fun info ()Ljava/lang/String;
|
||||
public abstract fun multiple ()Z
|
||||
public abstract fun required ()Z
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/ValueDef : java/lang/annotation/Annotation {
|
||||
public abstract fun allowed ()[Ljava/lang/String;
|
||||
public abstract fun def ()Ljava/lang/String;
|
||||
public abstract fun enumeration ()Ljava/lang/Class;
|
||||
public abstract fun type ()[Lspace/kscience/dataforge/values/ValueType;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/properties/Property$DefaultImpls {
|
||||
public static synthetic fun onChange$default (Lspace/kscience/dataforge/properties/Property;Ljava/lang/Object;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
|
||||
public static synthetic fun removeChangeListener$default (Lspace/kscience/dataforge/properties/Property;Ljava/lang/Object;ILjava/lang/Object;)V
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/properties/PropertyKt {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/properties/SchemePropertyKt {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/DfTypeKt {
|
||||
public static final fun getDfType (Lkotlin/reflect/KClass;)Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/Path : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker {
|
||||
@ -330,19 +309,14 @@ public final class space/kscience/dataforge/provider/PathToken {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/PathToken$Companion {
|
||||
public final fun parse (Ljava/lang/String;)Lspace/kscience/dataforge/provider/PathToken;
|
||||
public final fun parse (Ljava/lang/String;Z)Lspace/kscience/dataforge/provider/PathToken;
|
||||
public static synthetic fun parse$default (Lspace/kscience/dataforge/provider/PathToken$Companion;Ljava/lang/String;ZILjava/lang/Object;)Lspace/kscience/dataforge/provider/PathToken;
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/provider/Provider {
|
||||
public abstract fun content (Ljava/lang/String;)Ljava/util/Map;
|
||||
public abstract fun getDefaultChainTarget ()Ljava/lang/String;
|
||||
public abstract fun getDefaultTarget ()Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/Provider$DefaultImpls {
|
||||
public static fun content (Lspace/kscience/dataforge/provider/Provider;Ljava/lang/String;)Ljava/util/Map;
|
||||
public static fun getDefaultChainTarget (Lspace/kscience/dataforge/provider/Provider;)Ljava/lang/String;
|
||||
public static fun getDefaultTarget (Lspace/kscience/dataforge/provider/Provider;)Ljava/lang/String;
|
||||
public fun content (Ljava/lang/String;)Ljava/util/Map;
|
||||
public fun getDefaultChainTarget ()Ljava/lang/String;
|
||||
public fun getDefaultTarget ()Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/ProviderKt {
|
||||
|
@ -1,35 +1,24 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
description = "Context and provider definitions"
|
||||
|
||||
kscience {
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
useCoroutines()
|
||||
}
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
val commonMain by getting{
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
}
|
||||
}
|
||||
jvmMain {
|
||||
dependencies {
|
||||
api(kotlin("reflect"))
|
||||
api("org.slf4j:slf4j-api:1.7.30")
|
||||
}
|
||||
}
|
||||
jsMain {
|
||||
dependencies {
|
||||
|
||||
}
|
||||
}
|
||||
useSerialization()
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
}
|
||||
dependencies(jvmMain){
|
||||
api(kotlin("reflect"))
|
||||
api("org.slf4j:slf4j-api:1.7.30")
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.DEVELOPMENT
|
||||
readme {
|
||||
maturity = space.kscience.gradle.Maturity.DEVELOPMENT
|
||||
}
|
@ -1,6 +1,7 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.properties.ReadOnlyProperty
|
||||
@ -11,6 +12,8 @@ public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plu
|
||||
private var _context: Context? = null
|
||||
private val dependencies = HashMap<PluginFactory<*>, Meta>()
|
||||
|
||||
override val isAttached: Boolean get() = _context != null
|
||||
|
||||
override val context: Context
|
||||
get() = _context ?: error("Plugin $tag is not attached")
|
||||
|
||||
@ -22,21 +25,33 @@ public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plu
|
||||
this._context = null
|
||||
}
|
||||
|
||||
final override fun dependsOn(): Map<PluginFactory<*>, Meta> = dependencies
|
||||
override fun dependsOn(): Map<PluginFactory<*>, Meta> = dependencies
|
||||
|
||||
protected fun <P : Plugin> require(
|
||||
factory: PluginFactory<P>,
|
||||
type: KClass<P>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): ReadOnlyProperty<AbstractPlugin, P> {
|
||||
dependencies[factory] = meta
|
||||
return PluginDependencyDelegate(factory, type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin
|
||||
*/
|
||||
protected fun <P : Plugin> require(factory: PluginFactory<P>, meta: Meta = Meta.EMPTY): ReadOnlyProperty<AbstractPlugin, P> {
|
||||
dependencies[factory] = meta
|
||||
return PluginDependencyDelegate(factory.type)
|
||||
}
|
||||
protected inline fun <reified P : Plugin> require(
|
||||
factory: PluginFactory<P>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): ReadOnlyProperty<AbstractPlugin, P> = require(factory, P::class, meta)
|
||||
}
|
||||
|
||||
public fun <T : Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name to it }
|
||||
public fun <T : Named> Collection<T>.associateByName(): Map<Name, T> = associate { it.name to it }
|
||||
|
||||
private class PluginDependencyDelegate<P : Plugin>(val type: KClass<out P>) : ReadOnlyProperty<AbstractPlugin, P> {
|
||||
private class PluginDependencyDelegate<P : Plugin>(val factory: PluginFactory<P>, val type: KClass<P>) :
|
||||
ReadOnlyProperty<AbstractPlugin, P> {
|
||||
@OptIn(DFInternal::class)
|
||||
override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P {
|
||||
return thisRef.context.plugins[type] ?: error("Plugin with type $type not found")
|
||||
if (!thisRef.isAttached) error("Plugin dependency must not be called eagerly during initialization.")
|
||||
return thisRef.context.plugins.getByType(type, factory.tag) ?: error("Plugin ${factory.tag} not found")
|
||||
}
|
||||
}
|
@ -3,11 +3,10 @@ package space.kscience.dataforge.context
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import space.kscience.dataforge.meta.Laminate
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.meta.itemSequence
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.misc.ThreadSafe
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.provider.Provider
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
@ -25,35 +24,37 @@ import kotlin.coroutines.CoroutineContext
|
||||
public open class Context internal constructor(
|
||||
final override val name: Name,
|
||||
public val parent: Context?,
|
||||
plugins: Set<Plugin>, // set of unattached plugins
|
||||
meta: Meta,
|
||||
) : Named, MetaRepr, Provider, CoroutineScope {
|
||||
|
||||
/**
|
||||
* Context properties. Working as substitute for environment variables
|
||||
*/
|
||||
private val properties: Laminate = if (parent == null) {
|
||||
public val properties: Laminate = if (parent == null) {
|
||||
Laminate(meta)
|
||||
} else {
|
||||
Laminate(meta, parent.properties)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* A [PluginManager] for current context
|
||||
*/
|
||||
public val plugins: PluginManager by lazy { PluginManager(this) }
|
||||
public val plugins: PluginManager by lazy { PluginManager(this, plugins) }
|
||||
|
||||
override val defaultTarget: String get() = Plugin.TARGET
|
||||
|
||||
public fun content(target: String, inherit: Boolean): Map<Name, Any> {
|
||||
return if (inherit) {
|
||||
when (target) {
|
||||
PROPERTY_TARGET -> properties.itemSequence().toMap()
|
||||
PROPERTY_TARGET -> properties.nodeSequence().toMap()
|
||||
Plugin.TARGET -> plugins.list(true).associateBy { it.name }
|
||||
else -> emptyMap()
|
||||
}
|
||||
} else {
|
||||
when (target) {
|
||||
PROPERTY_TARGET -> properties.layers.firstOrNull()?.itemSequence()?.toMap() ?: emptyMap()
|
||||
PROPERTY_TARGET -> properties.layers.firstOrNull()?.nodeSequence()?.toMap() ?: emptyMap()
|
||||
Plugin.TARGET -> plugins.list(false).associateBy { it.name }
|
||||
else -> emptyMap()
|
||||
}
|
||||
@ -68,18 +69,35 @@ public open class Context internal constructor(
|
||||
}
|
||||
}
|
||||
|
||||
private val childrenContexts = HashMap<Name, Context>()
|
||||
|
||||
/**
|
||||
* Detach all plugins and terminate context
|
||||
* Get and validate existing context or build and register a new child context.
|
||||
* @param name the relative (tail) name of the new context. If null, uses context hash code as a marker.
|
||||
*/
|
||||
@OptIn(DFExperimental::class)
|
||||
@ThreadSafe
|
||||
public fun buildContext(name: Name? = null, block: ContextBuilder.() -> Unit = {}): Context {
|
||||
val existing = name?.let { childrenContexts[name] }
|
||||
return existing?.modify(block) ?: ContextBuilder(this, name).apply(block).build().also {
|
||||
childrenContexts[it.name] = it
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detach all plugins, and close child contexts
|
||||
*/
|
||||
public open fun close() {
|
||||
//recursively closed child context
|
||||
childrenContexts.forEach { it.value.close() }
|
||||
//detach all plugins
|
||||
plugins.forEach { it.detach() }
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
"parent" to parent?.name
|
||||
"properties" put properties.layers.firstOrNull()
|
||||
"plugins" put plugins.map { it.toMeta() }
|
||||
properties.layers.firstOrNull()?.let { set("properties", it) }
|
||||
"plugins" putIndexed plugins.map { it.toMeta() }
|
||||
}
|
||||
|
||||
public companion object {
|
||||
@ -87,9 +105,6 @@ public open class Context internal constructor(
|
||||
}
|
||||
}
|
||||
|
||||
public fun Context(name: String, parent: Context = Global, block: ContextBuilder.() -> Unit = {}): Context =
|
||||
Global.context(name, parent, block)
|
||||
|
||||
/**
|
||||
* The interface for something that encapsulated in context
|
||||
*
|
||||
|
@ -1,11 +1,15 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFBuilder
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.NameToken
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.plus
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.collections.set
|
||||
@ -14,11 +18,15 @@ import kotlin.collections.set
|
||||
* A convenience builder for context
|
||||
*/
|
||||
@DFBuilder
|
||||
public class ContextBuilder(private val parent: Context = Global, public var name: String = "@anonymous") {
|
||||
private val factories = HashMap<PluginFactory<*>, Meta>()
|
||||
private var meta = MetaBuilder()
|
||||
public class ContextBuilder internal constructor(
|
||||
private val parent: Context,
|
||||
public val name: Name? = null,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
) {
|
||||
internal val factories = HashMap<PluginFactory<*>, Meta>()
|
||||
internal var meta = meta.toMutableMeta()
|
||||
|
||||
public fun properties(action: MetaBuilder.() -> Unit) {
|
||||
public fun properties(action: MutableMeta.() -> Unit) {
|
||||
meta.action()
|
||||
}
|
||||
|
||||
@ -27,24 +35,77 @@ public class ContextBuilder(private val parent: Context = Global, public var nam
|
||||
parent.gatherInSequence<PluginFactory<*>>(PluginFactory.TYPE).values
|
||||
.find { it.tag.matches(tag) } ?: error("Can't resolve plugin factory for $tag")
|
||||
|
||||
public fun plugin(tag: PluginTag, metaBuilder: MetaBuilder.() -> Unit = {}) {
|
||||
public fun plugin(tag: PluginTag, mutableMeta: MutableMeta.() -> Unit = {}) {
|
||||
val factory = findPluginFactory(tag)
|
||||
factories[factory] = Meta(metaBuilder)
|
||||
factories[factory] = Meta(mutableMeta)
|
||||
}
|
||||
|
||||
public fun plugin(factory: PluginFactory<*>, metaBuilder: MetaBuilder.() -> Unit = {}) {
|
||||
factories[factory] = Meta(metaBuilder)
|
||||
public fun plugin(factory: PluginFactory<*>, meta: Meta) {
|
||||
factories[factory] = meta
|
||||
}
|
||||
|
||||
public fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) {
|
||||
public fun plugin(factory: PluginFactory<*>, mutableMeta: MutableMeta.() -> Unit = {}) {
|
||||
factories[factory] = Meta(mutableMeta)
|
||||
}
|
||||
|
||||
public fun plugin(name: String, group: String = "", version: String = "", action: MutableMeta.() -> Unit = {}) {
|
||||
plugin(PluginTag(name, group, version), action)
|
||||
}
|
||||
|
||||
public fun build(): Context {
|
||||
return Context(name.toName(), parent, meta.seal()).apply {
|
||||
factories.forEach { (factory, meta) ->
|
||||
plugins.load(factory, meta)
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Add de-facto existing plugin as a dependency
|
||||
*/
|
||||
public fun plugin(plugin: Plugin) {
|
||||
plugin(DeFactoPluginFactory(plugin))
|
||||
}
|
||||
|
||||
public fun build(): Context {
|
||||
val contextName = name ?: NameToken("@auto",hashCode().toUInt().toString(16)).asName()
|
||||
val plugins = HashMap<PluginTag, Plugin>()
|
||||
|
||||
fun addPlugin(factory: PluginFactory<*>, meta: Meta) {
|
||||
val existing = plugins[factory.tag]
|
||||
// Add if does not exist
|
||||
if (existing == null) {
|
||||
//TODO bypass if parent already has plugin with given meta?
|
||||
val plugin = factory.build(parent, meta)
|
||||
|
||||
for ((depFactory, deoMeta) in plugin.dependsOn()) {
|
||||
addPlugin(depFactory, deoMeta)
|
||||
}
|
||||
|
||||
parent.logger.info { "Loading plugin ${plugin.name} into $contextName" }
|
||||
plugins[plugin.tag] = plugin
|
||||
} else if (existing.meta != meta) {
|
||||
error("Plugin with tag ${factory.tag} and meta $meta already exists in $contextName")
|
||||
}
|
||||
//bypass if exists with the same meta
|
||||
}
|
||||
|
||||
factories.forEach { (factory, meta) ->
|
||||
addPlugin(factory, meta)
|
||||
}
|
||||
|
||||
return Context(contextName, parent, plugins.values.toSet(), meta.seal())
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if current context contains all plugins required by the builder and return it does or forks to a new context
|
||||
* if it does not.
|
||||
*/
|
||||
@DFExperimental
|
||||
public fun Context.modify(block: ContextBuilder.() -> Unit): Context {
|
||||
|
||||
fun Context.contains(factory: PluginFactory<*>, meta: Meta): Boolean {
|
||||
val loaded = plugins[factory.tag] ?: return false
|
||||
return loaded.meta == meta
|
||||
}
|
||||
|
||||
val builder = ContextBuilder(this, name + "mod", properties).apply(block)
|
||||
val requiresFork = builder.factories.any { (factory, meta) ->
|
||||
!contains(factory, meta)
|
||||
} || ((properties as Meta) == builder.meta)
|
||||
|
||||
return if (requiresFork) builder.build() else this
|
||||
}
|
@ -2,6 +2,11 @@ package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
|
||||
public interface Factory<out T : Any> {
|
||||
public operator fun invoke(meta: Meta = Meta.EMPTY, context: Context = Global): T
|
||||
}
|
||||
public fun interface Factory<out T> {
|
||||
public fun build(context: Context, meta: Meta): T
|
||||
}
|
||||
|
||||
public operator fun <T> Factory<T>.invoke(
|
||||
meta: Meta = Meta.EMPTY,
|
||||
context: Context = Global,
|
||||
): T = build(context, meta)
|
@ -1,55 +1,24 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import kotlinx.coroutines.GlobalScope
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.CoroutineName
|
||||
import kotlinx.coroutines.Job
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.native.concurrent.ThreadLocal
|
||||
|
||||
internal expect val globalLoggerFactory: PluginFactory<out LogManager>
|
||||
internal expect fun getGlobalLoggerFactory(): PluginFactory<out LogManager>
|
||||
|
||||
/**
|
||||
* A global root context. Closing [Global] terminates the framework.
|
||||
*/
|
||||
@ThreadLocal
|
||||
public object Global : Context("GLOBAL".asName(), null, Meta.EMPTY) {
|
||||
private object GlobalContext : Context("GLOBAL".asName(), null, emptySet(), Meta.EMPTY) {
|
||||
override val coroutineContext: CoroutineContext = Job() + CoroutineName("GlobalContext")
|
||||
}
|
||||
|
||||
override val coroutineContext: CoroutineContext = GlobalScope.coroutineContext + SupervisorJob()
|
||||
public val Global: Context get() = GlobalContext
|
||||
|
||||
/**
|
||||
* The default logging manager
|
||||
*/
|
||||
public val logger: LogManager by lazy { globalLoggerFactory.invoke(context = this).apply { attach(this@Global) } }
|
||||
|
||||
/**
|
||||
* Closing all contexts
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
override fun close() {
|
||||
logger.info { "Shutting down GLOBAL" }
|
||||
for (ctx in contextRegistry.values) {
|
||||
ctx.close()
|
||||
}
|
||||
super.close()
|
||||
}
|
||||
|
||||
private val contextRegistry = HashMap<String, Context>()
|
||||
|
||||
/**
|
||||
* Get previously built context
|
||||
*
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
public fun getContext(name: String): Context? {
|
||||
return contextRegistry[name]
|
||||
}
|
||||
|
||||
public fun context(name: String, parent: Context = this, block: ContextBuilder.() -> Unit = {}): Context =
|
||||
ContextBuilder(parent, name).apply(block).build().also {
|
||||
contextRegistry[name] = it
|
||||
}
|
||||
|
||||
}
|
||||
public fun Context(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context =
|
||||
Global.buildContext(name?.parseAsName(), block)
|
@ -1,14 +1,22 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
|
||||
public interface Logable {
|
||||
public fun log(name: Name, tag: String, body: () -> String)
|
||||
public fun interface Logger {
|
||||
public fun log(tag: String, body: () -> String)
|
||||
}
|
||||
|
||||
public interface LogManager : Plugin, Logable {
|
||||
public interface LogManager : Plugin, Logger {
|
||||
public fun logger(name: Name): Logger
|
||||
|
||||
public val defaultLogger: Logger
|
||||
|
||||
override fun log(tag: String, body: () -> String): Unit = defaultLogger.log(tag, body)
|
||||
|
||||
public fun log(name: Name, tag: String, body: () -> String): Unit = logger(name).log(tag, body)
|
||||
|
||||
public companion object {
|
||||
public const val TRACE: String = "TRACE"
|
||||
@ -19,11 +27,11 @@ public interface LogManager : Plugin, Logable {
|
||||
}
|
||||
}
|
||||
|
||||
public fun Logable.trace(name: Name = Name.EMPTY, body: () -> String): Unit = log(name, LogManager.TRACE, body)
|
||||
public fun Logable.info(name: Name = Name.EMPTY, body: () -> String): Unit = log(name, LogManager.INFO, body)
|
||||
public fun Logable.debug(name: Name = Name.EMPTY, body: () -> String): Unit = log(name, LogManager.DEBUG, body)
|
||||
public fun Logable.warn(name: Name = Name.EMPTY, body: () -> String): Unit = log(name, LogManager.WARNING, body)
|
||||
public fun Logable.error(name: Name = Name.EMPTY, body: () -> String): Unit = log(name, LogManager.ERROR, body)
|
||||
public fun Logger.trace(body: () -> String): Unit = log(LogManager.TRACE, body)
|
||||
public fun Logger.info(body: () -> String): Unit = log(LogManager.INFO, body)
|
||||
public fun Logger.debug(body: () -> String): Unit = log(LogManager.DEBUG, body)
|
||||
public fun Logger.warn(body: () -> String): Unit = log(LogManager.WARNING, body)
|
||||
public fun Logger.error(body: () -> String): Unit = log(LogManager.ERROR, body)
|
||||
|
||||
internal val (() -> String).safe: String
|
||||
get() = try {
|
||||
@ -33,30 +41,47 @@ internal val (() -> String).safe: String
|
||||
}
|
||||
|
||||
|
||||
public fun Logable.error(throwable: Throwable?, name: Name = Name.EMPTY, body: () -> String): Unit =
|
||||
log(name, LogManager.ERROR) {
|
||||
buildString {
|
||||
appendLine(body())
|
||||
throwable?.let { appendLine(throwable.stackTraceToString()) }
|
||||
}
|
||||
public fun Logger.error(throwable: Throwable?, body: () -> String): Unit = log(LogManager.ERROR) {
|
||||
buildString {
|
||||
appendLine(body())
|
||||
throwable?.let { appendLine(throwable.stackTraceToString()) }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class DefaultLogManager : AbstractPlugin(), LogManager {
|
||||
|
||||
override fun logger(name: Name): Logger = Logger { tag, body ->
|
||||
val message: String = body.safe
|
||||
println("$tag $name: [${context.name}] $message")
|
||||
}
|
||||
|
||||
override val defaultLogger: Logger = logger(Name.EMPTY)
|
||||
|
||||
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
public companion object : PluginFactory<DefaultLogManager> {
|
||||
override fun build(context: Context, meta: Meta): DefaultLogManager = DefaultLogManager()
|
||||
|
||||
override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.default")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Context log manager inherited from parent
|
||||
*/
|
||||
public val Context.logger: LogManager
|
||||
get() = plugins.find(inherit = true) { it is LogManager } as? LogManager ?: Global.logger
|
||||
get() = plugins.find(inherit = true) { it is LogManager } as? LogManager
|
||||
?: getGlobalLoggerFactory().build(context = Global, meta = Meta.EMPTY).apply { attach(Global) }
|
||||
|
||||
/**
|
||||
* The named proxy logger for a context member
|
||||
*/
|
||||
public val ContextAware.logger: Logable
|
||||
public val ContextAware.logger: Logger
|
||||
get() = if (this is Named) {
|
||||
object : Logable {
|
||||
val contextLog = context.logger
|
||||
override fun log(name: Name, tag: String, body: () -> String) {
|
||||
contextLog.log(this@logger.name + name, tag, body)
|
||||
}
|
||||
Logger { tag, body ->
|
||||
context.logger.log(this@logger.name + name, tag, body)
|
||||
}
|
||||
} else {
|
||||
context.logger
|
||||
|
@ -3,10 +3,10 @@ package space.kscience.dataforge.context
|
||||
import space.kscience.dataforge.context.Plugin.Companion.TARGET
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import space.kscience.dataforge.provider.Provider
|
||||
|
||||
/**
|
||||
@ -18,7 +18,7 @@ import space.kscience.dataforge.provider.Provider
|
||||
*
|
||||
* create - configure - attach - detach - destroy
|
||||
*/
|
||||
@Type(TARGET)
|
||||
@DfId(TARGET)
|
||||
public interface Plugin : Named, ContextAware, Provider, MetaRepr {
|
||||
|
||||
/**
|
||||
@ -31,7 +31,7 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
|
||||
/**
|
||||
* The name of this plugin ignoring version and group
|
||||
*/
|
||||
override val name: Name get() = tag.name.toName()
|
||||
override val name: Name get() = tag.name.parseAsName()
|
||||
|
||||
/**
|
||||
* Plugin dependencies which are required to attach this plugin. Plugin
|
||||
@ -53,6 +53,8 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
|
||||
*/
|
||||
public fun detach()
|
||||
|
||||
public val isAttached: Boolean
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
"context" put context.name.toString()
|
||||
"type" to this::class.simpleName
|
||||
@ -63,5 +65,4 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
|
||||
public companion object {
|
||||
public const val TARGET: String = "plugin"
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,57 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
|
||||
/**
|
||||
* A convenience factory to build simple plugins
|
||||
*/
|
||||
public class PluginBuilder(
|
||||
name: String,
|
||||
group: String = "",
|
||||
version: String = "",
|
||||
) {
|
||||
public val tag: PluginTag = PluginTag(name, group, version)
|
||||
|
||||
private val content = HashMap<String, MutableMap<Name, Any>>()
|
||||
private val dependencies = HashMap<PluginFactory<*>, Meta>()
|
||||
|
||||
public fun requires(
|
||||
factory: PluginFactory<*>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
) {
|
||||
dependencies[factory] = meta
|
||||
}
|
||||
|
||||
public fun provides(target: String, items: Map<Name, Any>) {
|
||||
content.getOrPut(target) { HashMap() }.putAll(items)
|
||||
}
|
||||
|
||||
public fun provides(target: String, vararg items: Named) {
|
||||
provides(target, items.associateBy { it.name })
|
||||
}
|
||||
|
||||
public fun build(): PluginFactory<*> {
|
||||
|
||||
return object : PluginFactory<Plugin> {
|
||||
override val tag: PluginTag get() = this@PluginBuilder.tag
|
||||
|
||||
override fun build(context: Context, meta: Meta): Plugin = object : AbstractPlugin() {
|
||||
override val tag: PluginTag get() = this@PluginBuilder.tag
|
||||
|
||||
override fun content(target: String): Map<Name, Any> = this@PluginBuilder.content[target] ?: emptyMap()
|
||||
|
||||
override fun dependsOn(): Map<PluginFactory<*>, Meta> = this@PluginBuilder.dependencies
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public fun PluginFactory(
|
||||
name: String,
|
||||
group: String = "",
|
||||
version: String = "",
|
||||
block: PluginBuilder.() -> Unit,
|
||||
): PluginFactory<*> = PluginBuilder(name, group, version).apply(block).build()
|
@ -1,14 +1,21 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import kotlin.reflect.KClass
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
|
||||
@Type(PluginFactory.TYPE)
|
||||
@DfId(PluginFactory.TYPE)
|
||||
public interface PluginFactory<T : Plugin> : Factory<T> {
|
||||
public val tag: PluginTag
|
||||
public val type: KClass<out T>
|
||||
|
||||
public companion object {
|
||||
public const val TYPE: String = "pluginFactory"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Plugin factory created for the specific actual plugin
|
||||
*/
|
||||
internal class DeFactoPluginFactory<T : Plugin>(val plugin: T) : PluginFactory<T> {
|
||||
override fun build(context: Context, meta: Meta): T = plugin
|
||||
override val tag: PluginTag get() = plugin.tag
|
||||
}
|
||||
|
@ -1,8 +1,10 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.plus
|
||||
import kotlin.reflect.KClass
|
||||
import kotlin.reflect.cast
|
||||
|
||||
|
||||
/**
|
||||
@ -11,14 +13,10 @@ import kotlin.reflect.KClass
|
||||
* @property context A context for this plugin manager
|
||||
* @author Alexander Nozik
|
||||
*/
|
||||
public class PluginManager(override val context: Context) : ContextAware, Iterable<Plugin> {
|
||||
|
||||
//TODO refactor to read-only container
|
||||
|
||||
/**
|
||||
* A set of loaded plugins
|
||||
*/
|
||||
private val plugins: HashSet<Plugin> = HashSet()
|
||||
public class PluginManager internal constructor(
|
||||
override val context: Context,
|
||||
private val plugins: Set<Plugin>,
|
||||
) : ContextAware, Iterable<Plugin> {
|
||||
|
||||
init {
|
||||
plugins.forEach { it.attach(context) }
|
||||
@ -68,76 +66,35 @@ public class PluginManager(override val context: Context) : ContextAware, Iterab
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
public operator fun <T : Any> get(type: KClass<out T>, tag: PluginTag? = null, recursive: Boolean = true): T? =
|
||||
find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T?
|
||||
@DFInternal
|
||||
public fun <T : Any> getByType(type: KClass<T>, tag: PluginTag? = null, inherit: Boolean = true): T? =
|
||||
find(inherit) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) }?.let { type.cast(it) }
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline operator fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
|
||||
get(T::class, tag, recursive)
|
||||
getByType(T::class, tag, recursive)
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline operator fun <reified T : Plugin> get(factory: PluginFactory<T>, recursive: Boolean = true): T? =
|
||||
get(factory.type, factory.tag, recursive)
|
||||
|
||||
/**
|
||||
* Load given plugin into this manager and return loaded instance.
|
||||
* Throw error if plugin of the same type and tag already exists in manager.
|
||||
*
|
||||
* @param plugin
|
||||
* @return
|
||||
*/
|
||||
public fun <T : Plugin> load(plugin: T): T {
|
||||
if (get(plugin::class, plugin.tag, recursive = false) != null) {
|
||||
error("Plugin with tag ${plugin.tag} already exists in ${context.name}")
|
||||
} else {
|
||||
for ((factory, meta) in plugin.dependsOn()) {
|
||||
fetch(factory, meta, true)
|
||||
}
|
||||
|
||||
Global.logger.info { "Loading plugin ${plugin.name} into ${context.name}" }
|
||||
plugin.attach(context)
|
||||
plugins.add(plugin)
|
||||
return plugin
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a plugin using its factory
|
||||
*/
|
||||
public fun <T : Plugin> load(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T =
|
||||
load(factory(meta, context))
|
||||
|
||||
public fun <T : Plugin> load(factory: PluginFactory<T>, metaBuilder: MetaBuilder.() -> Unit): T =
|
||||
load(factory, Meta(metaBuilder))
|
||||
|
||||
/**
|
||||
* Remove a plugin from [PluginManager]
|
||||
*/
|
||||
public fun remove(plugin: Plugin) {
|
||||
if (plugins.contains(plugin)) {
|
||||
Global.logger.info { "Removing plugin ${plugin.name} from ${context.name}" }
|
||||
plugin.detach()
|
||||
plugins.remove(plugin)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an existing plugin with given meta or load new one using provided factory
|
||||
*/
|
||||
public fun <T : Plugin> fetch(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY, recursive: Boolean = true): T {
|
||||
val loaded = get(factory.type, factory.tag, recursive)
|
||||
return when {
|
||||
loaded == null -> load(factory(meta, context))
|
||||
loaded.meta == meta -> loaded // if meta is the same, return existing plugin
|
||||
else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.")
|
||||
}
|
||||
}
|
||||
|
||||
public fun <T : Plugin> fetch(
|
||||
factory: PluginFactory<T>,
|
||||
recursive: Boolean = true,
|
||||
metaBuilder: MetaBuilder.() -> Unit,
|
||||
): T = fetch(factory, Meta(metaBuilder), recursive)
|
||||
getByType(T::class, factory.tag, recursive)
|
||||
|
||||
override fun iterator(): Iterator<Plugin> = plugins.iterator()
|
||||
}
|
||||
|
||||
}
|
||||
/**
|
||||
* Fetch a plugin with given meta from the context. If the plugin (with given meta) is already registered, it is returned.
|
||||
* Otherwise, new child context with the plugin is created. In the later case the context could be retrieved from the plugin.
|
||||
*/
|
||||
public inline fun <reified T : Plugin> Context.request(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T {
|
||||
val existing = plugins[factory]
|
||||
return if (existing != null && existing.meta == meta) existing
|
||||
else {
|
||||
buildContext(name = this@request.name + factory.tag.name) {
|
||||
plugin(factory, meta)
|
||||
}.plugins[factory]!!
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated("Replace with request", ReplaceWith("request(factory, meta)"))
|
||||
public inline fun <reified T : Plugin> Context.fetch(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T =
|
||||
request(factory, meta)
|
@ -1,5 +1,6 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
|
||||
@ -9,6 +10,7 @@ import space.kscience.dataforge.meta.MetaRepr
|
||||
*
|
||||
* @author Alexander Nozik
|
||||
*/
|
||||
@Serializable
|
||||
public data class PluginTag(
|
||||
val name: String,
|
||||
val group: String = "",
|
||||
|
@ -1,34 +0,0 @@
|
||||
package space.kscience.dataforge.properties
|
||||
|
||||
import space.kscience.dataforge.meta.Config
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.set
|
||||
import space.kscience.dataforge.meta.transformations.MetaConverter
|
||||
import space.kscience.dataforge.meta.transformations.nullableItemToObject
|
||||
import space.kscience.dataforge.meta.transformations.nullableObjectToMetaItem
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
|
||||
@DFExperimental
|
||||
public class ConfigProperty<T : Any>(
|
||||
public val config: Config,
|
||||
public val name: Name,
|
||||
public val converter: MetaConverter<T>,
|
||||
) : Property<T?> {
|
||||
|
||||
override var value: T?
|
||||
get() = converter.nullableItemToObject(config[name])
|
||||
set(value) {
|
||||
config[name] = converter.nullableObjectToMetaItem(value)
|
||||
}
|
||||
|
||||
override fun onChange(owner: Any?, callback: (T?) -> Unit) {
|
||||
config.onChange(owner) { name, oldItem, newItem ->
|
||||
if (name == this.name && oldItem != newItem) callback(converter.nullableItemToObject(newItem))
|
||||
}
|
||||
}
|
||||
|
||||
override fun removeChangeListener(owner: Any?) {
|
||||
config.removeListener(owner)
|
||||
}
|
||||
}
|
@ -0,0 +1,35 @@
|
||||
package space.kscience.dataforge.properties
|
||||
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.ObservableMutableMeta
|
||||
import space.kscience.dataforge.meta.transformations.MetaConverter
|
||||
import space.kscience.dataforge.meta.transformations.nullableMetaToObject
|
||||
import space.kscience.dataforge.meta.transformations.nullableObjectToMeta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
|
||||
@DFExperimental
|
||||
public class MetaProperty<T : Any>(
|
||||
public val meta: ObservableMutableMeta,
|
||||
public val name: Name,
|
||||
public val converter: MetaConverter<T>,
|
||||
) : Property<T?> {
|
||||
|
||||
override var value: T?
|
||||
get() = converter.nullableMetaToObject(meta[name])
|
||||
set(value) {
|
||||
meta[name] = converter.nullableObjectToMeta(value) ?: Meta.EMPTY
|
||||
}
|
||||
|
||||
override fun onChange(owner: Any?, callback: (T?) -> Unit) {
|
||||
meta.onChange(owner) { name ->
|
||||
if (name.startsWith(this@MetaProperty.name)) callback(converter.nullableMetaToObject(this[name]))
|
||||
}
|
||||
}
|
||||
|
||||
override fun removeChangeListener(owner: Any?) {
|
||||
meta.removeListener(owner)
|
||||
}
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.properties
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
@ -23,12 +22,12 @@ public fun <T> Property<T>.toFlow(): StateFlow<T> = MutableStateFlow(value).also
|
||||
}
|
||||
|
||||
/**
|
||||
* Reflect all changes in the [source] property onto this property
|
||||
* Reflect all changes in the [source] property onto this property. Does not reflect changes back.
|
||||
*
|
||||
* @return a mirroring job
|
||||
*/
|
||||
@DFExperimental
|
||||
public fun <T> Property<T>.mirror(source: Property<T>, scope: CoroutineScope) {
|
||||
public fun <T> Property<T>.mirror(source: Property<T>) {
|
||||
source.onChange(this) {
|
||||
this.value = it
|
||||
}
|
||||
|
@ -0,0 +1,31 @@
|
||||
package space.kscience.dataforge.properties
|
||||
|
||||
|
||||
import space.kscience.dataforge.meta.Scheme
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
import kotlin.reflect.KMutableProperty1
|
||||
|
||||
@DFExperimental
|
||||
public fun <S : Scheme, T : Any> S.property(property: KMutableProperty1<S, T?>): Property<T?> =
|
||||
object : Property<T?> {
|
||||
override var value: T?
|
||||
get() = property.get(this@property)
|
||||
set(value) {
|
||||
property.set(this@property, value)
|
||||
}
|
||||
|
||||
override fun onChange(owner: Any?, callback: (T?) -> Unit) {
|
||||
this@property.meta.onChange(this) { name ->
|
||||
if (name.startsWith(property.name.parseAsName(true))) {
|
||||
callback(property.get(this@property))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun removeChangeListener(owner: Any?) {
|
||||
this@property.meta.removeListener(this@property)
|
||||
}
|
||||
|
||||
}
|
@ -16,13 +16,15 @@
|
||||
package space.kscience.dataforge.provider
|
||||
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.jvm.JvmInline
|
||||
|
||||
/**
|
||||
* Path interface.
|
||||
*
|
||||
*/
|
||||
public inline class Path(public val tokens: List<PathToken>) : Iterable<PathToken> {
|
||||
@JvmInline
|
||||
public value class Path(public val tokens: List<PathToken>) : Iterable<PathToken> {
|
||||
|
||||
override fun iterator(): Iterator<PathToken> = tokens.iterator()
|
||||
|
||||
@ -31,11 +33,7 @@ public inline class Path(public val tokens: List<PathToken>) : Iterable<PathToke
|
||||
public companion object {
|
||||
public const val PATH_SEGMENT_SEPARATOR: String = "/"
|
||||
|
||||
public fun parse(path: String): Path {
|
||||
val head = path.substringBefore(PATH_SEGMENT_SEPARATOR)
|
||||
val tail = path.substringAfter(PATH_SEGMENT_SEPARATOR)
|
||||
return PathToken.parse(head).asPath() + parse(tail)
|
||||
}
|
||||
public fun parse(path: String): Path = Path(path.split(PATH_SEGMENT_SEPARATOR).map { PathToken.parse(it) })
|
||||
}
|
||||
}
|
||||
|
||||
@ -63,9 +61,10 @@ public data class PathToken(val name: Name, val target: String? = null) {
|
||||
|
||||
public companion object {
|
||||
public const val TARGET_SEPARATOR: String = "::"
|
||||
public fun parse(token: String): PathToken {
|
||||
|
||||
public fun parse(token: String, cache: Boolean = false): PathToken {
|
||||
val target = token.substringBefore(TARGET_SEPARATOR, "")
|
||||
val name = token.substringAfter(TARGET_SEPARATOR).toName()
|
||||
val name = token.substringAfter(TARGET_SEPARATOR).parseAsName(cache)
|
||||
if (target.contains("[")) TODO("target separators in queries are not supported")
|
||||
return PathToken(name, target)
|
||||
}
|
||||
|
@ -75,10 +75,8 @@ public inline fun <reified T : Any> Provider.provide(path: String, targetOverrid
|
||||
/**
|
||||
* Typed top level content
|
||||
*/
|
||||
public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> {
|
||||
return content(target).mapValues {
|
||||
type.safeCast(it.value) ?: error("The type of element $it is ${it::class} but $type is expected")
|
||||
}
|
||||
public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> = content(target).mapValues {
|
||||
type.safeCast(it.value) ?: error("The type of element ${it.value} is ${it.value::class} but $type is expected")
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1,8 +1,7 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.appendLeft
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.appendFirst
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -12,23 +11,22 @@ class ContextTest {
|
||||
override val tag get() = PluginTag("test")
|
||||
|
||||
override fun content(target: String): Map<Name, Any> {
|
||||
return when(target){
|
||||
"test" -> listOf("a", "b", "c.d").associate { it.toName() to it.toName() }
|
||||
return when (target) {
|
||||
"test" -> listOf("a", "b", "c.d").associate { Name.parse(it) to Name.parse(it) }
|
||||
else -> emptyMap()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testPluginManager() {
|
||||
val context = Global.context("test")
|
||||
context.plugins.load(DummyPlugin())
|
||||
//Global.plugins.load(DummyPlugin())
|
||||
val context = Context("test") {
|
||||
plugin(DummyPlugin())
|
||||
}
|
||||
val members = context.gather<Name>("test")
|
||||
assertEquals(3, members.count())
|
||||
members.forEach {
|
||||
assertEquals(it.key, it.value.appendLeft("test"))
|
||||
assertEquals(it.key, it.value.appendFirst("test"))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -0,0 +1,28 @@
|
||||
package space.kscience.dataforge.properties
|
||||
|
||||
import space.kscience.dataforge.meta.Scheme
|
||||
import space.kscience.dataforge.meta.SchemeSpec
|
||||
import space.kscience.dataforge.meta.int
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
internal class TestScheme : Scheme() {
|
||||
var a by int()
|
||||
var b by int()
|
||||
companion object : SchemeSpec<TestScheme>(::TestScheme)
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
class MetaPropertiesTest {
|
||||
@Test
|
||||
fun testBinding() {
|
||||
val scheme = TestScheme.empty()
|
||||
val a = scheme.property(TestScheme::a)
|
||||
val b = scheme.property(TestScheme::b)
|
||||
a.bind(b)
|
||||
scheme.a = 2
|
||||
assertEquals(2, scheme.b)
|
||||
assertEquals(2, b.value)
|
||||
}
|
||||
}
|
@ -0,0 +1,14 @@
|
||||
package space.kscience.dataforge.provider
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class PathTest {
|
||||
@Test
|
||||
fun testParse(){
|
||||
val nameString = "a.b.c.d"
|
||||
val pathString = "a.b/c.d"
|
||||
assertEquals(1, Path.parse(nameString).length)
|
||||
assertEquals(2, Path.parse(pathString).length)
|
||||
}
|
||||
}
|
@ -2,13 +2,14 @@ package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
public class ConsoleLogManager : AbstractPlugin(), LogManager {
|
||||
|
||||
override fun log(name: Name, tag: String, body: () -> String) {
|
||||
override fun logger(name: Name): Logger = Logger { tag, body ->
|
||||
val message: String = body.safe
|
||||
when (tag) {
|
||||
// TODO depends on https://youtrack.jetbrains.com/issue/KT-33595/
|
||||
LogManager.DEBUG -> console.asDynamic().debug("[${context.name}] $name: $message")
|
||||
LogManager.INFO -> console.info("[${context.name}] $name: $message")
|
||||
LogManager.WARNING -> console.warn("[${context.name}] $name: $message")
|
||||
LogManager.ERROR -> console.error("[${context.name}] $name: $message")
|
||||
@ -16,14 +17,16 @@ public class ConsoleLogManager : AbstractPlugin(), LogManager {
|
||||
}
|
||||
}
|
||||
|
||||
override val defaultLogger: Logger = logger(Name.EMPTY)
|
||||
|
||||
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
public companion object : PluginFactory<ConsoleLogManager> {
|
||||
override fun invoke(meta: Meta, context: Context): ConsoleLogManager = ConsoleLogManager()
|
||||
override fun build(context: Context, meta: Meta): ConsoleLogManager = ConsoleLogManager()
|
||||
|
||||
override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.jsConsole")
|
||||
override val type: KClass<out ConsoleLogManager> = ConsoleLogManager::class
|
||||
}
|
||||
}
|
||||
|
||||
internal actual val globalLoggerFactory: PluginFactory<out LogManager> = ConsoleLogManager
|
||||
internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = ConsoleLogManager
|
||||
|
@ -3,11 +3,10 @@ package space.kscience.dataforge.context
|
||||
import org.slf4j.LoggerFactory
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
public class SlfLogManager : AbstractPlugin(), LogManager {
|
||||
|
||||
override fun log(name: Name, tag: String, body: () -> String) {
|
||||
override fun logger(name: Name): Logger = Logger { tag, body ->
|
||||
val logger = LoggerFactory.getLogger("[${context.name}] $name") //KotlinLogging.logger("[${context.name}] $name")
|
||||
val message = body.safe
|
||||
when (tag) {
|
||||
@ -19,14 +18,15 @@ public class SlfLogManager : AbstractPlugin(), LogManager {
|
||||
}
|
||||
}
|
||||
|
||||
override val defaultLogger: Logger = logger(Name.EMPTY)
|
||||
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
public companion object : PluginFactory<SlfLogManager> {
|
||||
override fun invoke(meta: Meta, context: Context): SlfLogManager = SlfLogManager()
|
||||
override fun build(context: Context, meta: Meta): SlfLogManager = SlfLogManager()
|
||||
|
||||
override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.kotlinLogging")
|
||||
override val type: KClass<out SlfLogManager> = SlfLogManager::class
|
||||
}
|
||||
}
|
||||
|
||||
internal actual val globalLoggerFactory: PluginFactory<out LogManager> = SlfLogManager
|
||||
internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = SlfLogManager
|
||||
|
@ -16,35 +16,32 @@
|
||||
|
||||
package space.kscience.dataforge.descriptors
|
||||
|
||||
import space.kscience.dataforge.values.ValueType
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
@MustBeDocumented
|
||||
annotation class Attribute(
|
||||
val key: String,
|
||||
val value: String
|
||||
)
|
||||
|
||||
@MustBeDocumented
|
||||
annotation class Attributes(
|
||||
val attrs: Array<Attribute>
|
||||
)
|
||||
|
||||
@MustBeDocumented
|
||||
annotation class ItemDef(
|
||||
val info: String = "",
|
||||
val multiple: Boolean = false,
|
||||
val required: Boolean = false
|
||||
)
|
||||
|
||||
@Target(AnnotationTarget.PROPERTY)
|
||||
@MustBeDocumented
|
||||
annotation class ValueDef(
|
||||
val type: Array<ValueType> = [ValueType.STRING],
|
||||
val def: String = "",
|
||||
val allowed: Array<String> = [],
|
||||
val enumeration: KClass<*> = Any::class
|
||||
)
|
||||
//@MustBeDocumented
|
||||
//annotation class Attribute(
|
||||
// val key: String,
|
||||
// val value: String
|
||||
//)
|
||||
//
|
||||
//@MustBeDocumented
|
||||
//annotation class Attributes(
|
||||
// val attrs: Array<Attribute>
|
||||
//)
|
||||
//
|
||||
//@MustBeDocumented
|
||||
//annotation class ItemDef(
|
||||
// val info: String = "",
|
||||
// val multiple: Boolean = false,
|
||||
// val required: Boolean = false
|
||||
//)
|
||||
//
|
||||
//@Target(AnnotationTarget.PROPERTY)
|
||||
//@MustBeDocumented
|
||||
//annotation class ValueDef(
|
||||
// val type: Array<ValueType> = [ValueType.STRING],
|
||||
// val def: String = "",
|
||||
// val allowed: Array<String> = [],
|
||||
// val enumeration: KClass<*> = Any::class
|
||||
//)
|
||||
|
||||
///**
|
||||
// * Description text for meta property, node or whole object
|
||||
|
@ -1,30 +1,32 @@
|
||||
package space.kscience.dataforge.provider
|
||||
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.PluginBuilder
|
||||
import space.kscience.dataforge.context.gather
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.reflect.KClass
|
||||
import kotlin.reflect.full.findAnnotation
|
||||
|
||||
|
||||
@DFExperimental
|
||||
public val KClass<*>.dfType: String
|
||||
get() = findAnnotation<Type>()?.id ?: simpleName ?: ""
|
||||
public val KClass<*>.dfId: String
|
||||
get() = findAnnotation<DfId>()?.id ?: simpleName ?: ""
|
||||
|
||||
/**
|
||||
* Provide an object with given name inferring target from its type using [Type] annotation
|
||||
* Provide an object with given name inferring target from its type using [DfId] annotation
|
||||
*/
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> Provider.provideByType(name: String): T? {
|
||||
val target = T::class.dfType
|
||||
val target = T::class.dfId
|
||||
return provide(target, name)
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
|
||||
val target = T::class.dfType
|
||||
val target = T::class.dfId
|
||||
return top(target)
|
||||
}
|
||||
|
||||
@ -33,5 +35,15 @@ public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
|
||||
*/
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> Context.gather(inherit: Boolean = true): Map<Name, T> =
|
||||
gather<T>(T::class.dfType, inherit)
|
||||
gather<T>(T::class.dfId, inherit)
|
||||
|
||||
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> PluginBuilder.provides(items: Map<Name, T>) {
|
||||
provides(T::class.dfId, items)
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> PluginBuilder.provides(vararg items: Named) {
|
||||
provides(T::class.dfId, *items)
|
||||
}
|
||||
|
@ -1,25 +1,4 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
|
||||
public class NativeLogManager : AbstractPlugin(), LogManager {
|
||||
|
||||
override fun log(name: Name, tag: String, body: () -> String) {
|
||||
val message: String = body.safe
|
||||
println("[${context.name}] $name: $message")
|
||||
}
|
||||
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
public companion object : PluginFactory<NativeLogManager> {
|
||||
override fun invoke(meta: Meta, context: Context): NativeLogManager = NativeLogManager()
|
||||
|
||||
override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.native")
|
||||
override val type: KClass<out NativeLogManager> = NativeLogManager::class
|
||||
}
|
||||
}
|
||||
|
||||
internal actual val globalLoggerFactory: PluginFactory<out LogManager> = NativeLogManager
|
||||
internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = DefaultLogManager
|
||||
|
23
dataforge-data/README.md
Normal file
23
dataforge-data/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-data
|
||||
|
||||
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-data:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-data:0.7.0")
|
||||
}
|
||||
```
|
@ -1,23 +1,18 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
kscience{
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
useCoroutines()
|
||||
}
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
commonMain{
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
api(kotlin("reflect"))
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
api(kotlin("reflect"))
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
|
||||
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
|
||||
}
|
||||
|
@ -0,0 +1,65 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
import kotlin.reflect.KType
|
||||
|
||||
/**
|
||||
* Remove all values with keys starting with [name]
|
||||
*/
|
||||
internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
|
||||
val toRemove = keys.filter { it.startsWith(name) }
|
||||
toRemove.forEach(::remove)
|
||||
}
|
||||
|
||||
/**
|
||||
* An action that caches results on-demand and recalculates them on source push
|
||||
*/
|
||||
public abstract class AbstractAction<in T : Any, R : Any>(
|
||||
public val outputType: KType,
|
||||
) : Action<T, R> {
|
||||
|
||||
/**
|
||||
* Generate initial content of the output
|
||||
*/
|
||||
protected abstract fun DataSetBuilder<R>.generate(
|
||||
data: DataSet<T>,
|
||||
meta: Meta,
|
||||
)
|
||||
|
||||
/**
|
||||
* Update part of the data set when given [updateKey] is triggered by the source
|
||||
*/
|
||||
protected open fun DataSourceBuilder<R>.update(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
updateKey: Name,
|
||||
) {
|
||||
// By default, recalculate the whole dataset
|
||||
generate(dataSet, meta)
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
override fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
): DataSet<R> = if (dataSet is DataSource) {
|
||||
DataSource(outputType, dataSet){
|
||||
generate(dataSet, meta)
|
||||
|
||||
launch {
|
||||
dataSet.updates.collect { name ->
|
||||
update(dataSet, meta, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DataTree<R>(outputType) {
|
||||
generate(dataSet, meta)
|
||||
}
|
||||
}
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import space.kscience.dataforge.data.DataSet
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
@ -9,13 +8,12 @@ import space.kscience.dataforge.misc.DFExperimental
|
||||
* A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute].
|
||||
*/
|
||||
public interface Action<in T : Any, out R : Any> {
|
||||
|
||||
/**
|
||||
* Transform the data in the node, producing a new node. By default it is assumed that all calculations are lazy
|
||||
* Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy
|
||||
* so not actual computation is started at this moment.
|
||||
*
|
||||
* [scope] context used to compute the initial result, also it is used for updates propagation
|
||||
*/
|
||||
public suspend fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY, scope: CoroutineScope? = null): DataSet<R>
|
||||
public fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY): DataSet<R>
|
||||
|
||||
public companion object
|
||||
}
|
||||
@ -26,16 +24,17 @@ public interface Action<in T : Any, out R : Any> {
|
||||
public infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> {
|
||||
// TODO introduce composite action and add optimize by adding action to the list
|
||||
return object : Action<T, R> {
|
||||
override suspend fun execute(dataSet: DataSet<T>, meta: Meta, scope: CoroutineScope?): DataSet<R> {
|
||||
return action.execute(this@then.execute(dataSet, meta, scope), meta, scope)
|
||||
}
|
||||
|
||||
override fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
): DataSet<R> = action.execute(this@then.execute(dataSet, meta), meta)
|
||||
}
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
public suspend fun <T : Any, R : Any> DataSet<T>.transformWith(
|
||||
action: Action<T, R>,
|
||||
public operator fun <T : Any, R : Any> Action<T, R>.invoke(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
scope: CoroutineScope? = null,
|
||||
): DataSet<R> = action.execute(this, meta, scope)
|
||||
): DataSet<R> = execute(dataSet, meta)
|
||||
|
||||
|
@ -1,12 +1,8 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import kotlinx.coroutines.flow.map
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFBuilder
|
||||
@ -29,65 +25,71 @@ public data class ActionEnv(
|
||||
* Action environment
|
||||
*/
|
||||
@DFBuilder
|
||||
public class MapActionBuilder<T, R>(public var name: Name, public var meta: MetaBuilder, public val actionMeta: Meta) {
|
||||
public class MapActionBuilder<T, R>(
|
||||
public var name: Name,
|
||||
public var meta: MutableMeta,
|
||||
public val actionMeta: Meta,
|
||||
@PublishedApi internal var outputType: KType,
|
||||
) {
|
||||
|
||||
public lateinit var result: suspend ActionEnv.(T) -> R
|
||||
|
||||
/**
|
||||
* Set unsafe [outputType] for the resulting data. Be sure that it is correct.
|
||||
*/
|
||||
public fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(T) -> R1) {
|
||||
this.outputType = outputType
|
||||
result = f;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the result of goal
|
||||
*/
|
||||
public fun result(f: suspend ActionEnv.(T) -> R) {
|
||||
public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(T) -> R1) {
|
||||
outputType = typeOf<R1>()
|
||||
result = f;
|
||||
}
|
||||
}
|
||||
|
||||
@PublishedApi
|
||||
internal class MapAction<in T : Any, out R : Any>(
|
||||
private val outputType: KType,
|
||||
internal class MapAction<in T : Any, R : Any>(
|
||||
outputType: KType,
|
||||
private val block: MapActionBuilder<T, R>.() -> Unit,
|
||||
) : Action<T, R> {
|
||||
) : AbstractAction<T, R>(outputType) {
|
||||
|
||||
override suspend fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
scope: CoroutineScope?,
|
||||
): DataSet<R> {
|
||||
suspend fun mapOne(data: NamedData<T>): NamedData<R> {
|
||||
// Creating a new environment for action using **old** name, old meta and task meta
|
||||
val env = ActionEnv(data.name, data.meta, meta)
|
||||
private fun DataSetBuilder<R>.mapOne(name: Name, data: Data<T>, meta: Meta) {
|
||||
// Creating a new environment for action using **old** name, old meta and task meta
|
||||
val env = ActionEnv(name, data.meta, meta)
|
||||
|
||||
//applying transformation from builder
|
||||
val builder = MapActionBuilder<T, R>(
|
||||
data.name,
|
||||
data.meta.toMutableMeta(), // using data meta
|
||||
meta
|
||||
).apply(block)
|
||||
//applying transformation from builder
|
||||
val builder = MapActionBuilder<T, R>(
|
||||
name,
|
||||
data.meta.toMutableMeta(), // using data meta
|
||||
meta,
|
||||
outputType
|
||||
).apply(block)
|
||||
|
||||
//getting new name
|
||||
val newName = builder.name
|
||||
//getting new name
|
||||
val newName = builder.name
|
||||
|
||||
//getting new meta
|
||||
val newMeta = builder.meta.seal()
|
||||
//getting new meta
|
||||
val newMeta = builder.meta.seal()
|
||||
|
||||
@OptIn(DFInternal::class) val newData = Data(outputType, newMeta, dependencies = listOf(data)) {
|
||||
builder.result(env, data.await())
|
||||
}
|
||||
//setting the data node
|
||||
return newData.named(newName)
|
||||
@OptIn(DFInternal::class)
|
||||
val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) {
|
||||
builder.result(env, data.await())
|
||||
}
|
||||
//setting the data node
|
||||
data(newName, newData)
|
||||
}
|
||||
|
||||
val flow = dataSet.flow().map(::mapOne)
|
||||
override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
|
||||
data.forEach { mapOne(it.name, it.data, meta) }
|
||||
}
|
||||
|
||||
return ActiveDataTree(outputType) {
|
||||
populate(flow)
|
||||
scope?.launch {
|
||||
dataSet.updates.collect { name ->
|
||||
//clear old nodes
|
||||
remove(name)
|
||||
//collect new items
|
||||
populate(dataSet.flowChildren(name).map(::mapOne))
|
||||
}
|
||||
}
|
||||
}
|
||||
override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
|
||||
remove(updateKey)
|
||||
dataSet[updateKey]?.let { mapOne(updateKey, it, meta) }
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1,28 +1,34 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.flow
|
||||
import kotlinx.coroutines.flow.fold
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.misc.DFBuilder
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
|
||||
public class JoinGroup<T : Any, R : Any>(public var name: String, internal val set: DataSet<T>) {
|
||||
public class JoinGroup<T : Any, R : Any>(
|
||||
public var name: String,
|
||||
internal val set: DataSet<T>,
|
||||
@PublishedApi internal var outputType: KType,
|
||||
) {
|
||||
|
||||
public var meta: MetaBuilder = MetaBuilder()
|
||||
public var meta: MutableMeta = MutableMeta()
|
||||
|
||||
public lateinit var result: suspend ActionEnv.(Map<Name, T>) -> R
|
||||
public lateinit var result: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R
|
||||
|
||||
public fun result(f: suspend ActionEnv.(Map<Name, T>) -> R) {
|
||||
internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R1) {
|
||||
this.outputType = outputType
|
||||
this.result = f;
|
||||
}
|
||||
|
||||
public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R1) {
|
||||
outputType = typeOf<R1>()
|
||||
this.result = f;
|
||||
}
|
||||
|
||||
@ -30,31 +36,30 @@ public class JoinGroup<T : Any, R : Any>(public var name: String, internal val s
|
||||
|
||||
@DFBuilder
|
||||
public class ReduceGroupBuilder<T : Any, R : Any>(
|
||||
private val inputType: KType,
|
||||
private val scope: CoroutineScope,
|
||||
public val actionMeta: Meta,
|
||||
private val outputType: KType,
|
||||
) {
|
||||
private val groupRules: MutableList<suspend (DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
|
||||
private val groupRules: MutableList<(DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
|
||||
|
||||
/**
|
||||
* introduce grouping by meta value
|
||||
*/
|
||||
public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup<T, R>.() -> Unit) {
|
||||
groupRules += { node ->
|
||||
GroupRule.byMetaValue(scope, tag, defaultTag).gather(node).map {
|
||||
JoinGroup<T, R>(it.key, it.value).apply(action)
|
||||
GroupRule.byMetaValue(tag, defaultTag).gather(node).map {
|
||||
JoinGroup<T, R>(it.key, it.value, outputType).apply(action)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public fun group(
|
||||
groupName: String,
|
||||
filter: suspend (Name, Data<T>) -> Boolean,
|
||||
predicate: (Name, Meta) -> Boolean,
|
||||
action: JoinGroup<T, R>.() -> Unit,
|
||||
) {
|
||||
groupRules += { source ->
|
||||
listOf(
|
||||
JoinGroup<T, R>(groupName, source.filter(filter)).apply(action)
|
||||
JoinGroup<T, R>(groupName, source.filter(predicate), outputType).apply(action)
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -62,30 +67,27 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
|
||||
/**
|
||||
* Apply transformation to the whole node
|
||||
*/
|
||||
public fun result(resultName: String, f: suspend ActionEnv.(Map<Name, T>) -> R) {
|
||||
public fun result(resultName: String, f: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R) {
|
||||
groupRules += { node ->
|
||||
listOf(JoinGroup<T, R>(resultName, node).apply { result(f) })
|
||||
listOf(JoinGroup<T, R>(resultName, node, outputType).apply { result(outputType, f) })
|
||||
}
|
||||
}
|
||||
|
||||
internal suspend fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> {
|
||||
return groupRules.flatMap { it.invoke(input) }
|
||||
}
|
||||
internal fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> =
|
||||
groupRules.flatMap { it.invoke(input) }
|
||||
|
||||
}
|
||||
|
||||
@PublishedApi
|
||||
internal class ReduceAction<T : Any, R : Any>(
|
||||
private val inputType: KType,
|
||||
outputType: KType,
|
||||
private val action: ReduceGroupBuilder<T, R>.() -> Unit,
|
||||
) : CachingAction<T, R>(outputType) {
|
||||
//TODO optimize reduction. Currently the whole action recalculates on push
|
||||
) : AbstractAction<T, R>(outputType) {
|
||||
//TODO optimize reduction. Currently, the whole action recalculates on push
|
||||
|
||||
|
||||
override fun CoroutineScope.transform(set: DataSet<T>, meta: Meta, key: Name): Flow<NamedData<R>> = flow {
|
||||
ReduceGroupBuilder<T, R>(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group ->
|
||||
val dataFlow: Map<Name, Data<T>> = group.set.flow().fold(HashMap()) { acc, value ->
|
||||
override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
|
||||
ReduceGroupBuilder<T, R>(meta, outputType).apply(action).buildGroups(data).forEach { group ->
|
||||
val dataFlow: Map<Name, Data<T>> = group.set.asSequence().fold(HashMap()) { acc, value ->
|
||||
acc.apply {
|
||||
acc[value.name] = value.data
|
||||
}
|
||||
@ -95,13 +97,13 @@ internal class ReduceAction<T : Any, R : Any>(
|
||||
|
||||
val groupMeta = group.meta
|
||||
|
||||
val env = ActionEnv(groupName.toName(), groupMeta, meta)
|
||||
val env = ActionEnv(groupName.parseAsName(), groupMeta, meta)
|
||||
@OptIn(DFInternal::class) val res: Data<R> = dataFlow.reduceToData(
|
||||
outputType,
|
||||
group.outputType,
|
||||
meta = groupMeta
|
||||
) { group.result.invoke(env, it) }
|
||||
|
||||
emit(res.named(env.name))
|
||||
data(env.name, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -110,7 +112,6 @@ internal class ReduceAction<T : Any, R : Any>(
|
||||
* A one-to-one mapping action
|
||||
*/
|
||||
@DFExperimental
|
||||
@Suppress("FunctionName")
|
||||
public inline fun <reified T : Any, reified R : Any> Action.Companion.reduce(
|
||||
noinline builder: ReduceGroupBuilder<T, R>.() -> Unit,
|
||||
): Action<T, R> = ReduceAction(typeOf<T>(), typeOf<R>(), builder)
|
||||
): Action<T, R> = ReduceAction(typeOf<R>(), builder)
|
||||
|
@ -1,18 +1,13 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.FlowPreview
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Laminate
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.collections.set
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
@ -20,10 +15,15 @@ import kotlin.reflect.typeOf
|
||||
|
||||
public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val meta: Meta) {
|
||||
|
||||
public class FragmentRule<T : Any, R : Any>(public val name: Name, public var meta: MetaBuilder) {
|
||||
public class FragmentRule<T : Any, R : Any>(
|
||||
public val name: Name,
|
||||
public var meta: MutableMeta,
|
||||
@PublishedApi internal var outputType: KType,
|
||||
) {
|
||||
public lateinit var result: suspend (T) -> R
|
||||
|
||||
public fun result(f: suspend (T) -> R) {
|
||||
public inline fun <reified R1 : R> result(noinline f: suspend (T) -> R1) {
|
||||
this.outputType = typeOf<R1>()
|
||||
result = f;
|
||||
}
|
||||
}
|
||||
@ -36,7 +36,7 @@ public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val me
|
||||
* @param rule the rule to transform fragment name and meta using
|
||||
*/
|
||||
public fun fragment(name: String, rule: FragmentRule<T, R>.() -> Unit) {
|
||||
fragments[name.toName()] = rule
|
||||
fragments[name.parseAsName()] = rule
|
||||
}
|
||||
}
|
||||
|
||||
@ -45,52 +45,48 @@ public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val me
|
||||
*/
|
||||
@PublishedApi
|
||||
internal class SplitAction<T : Any, R : Any>(
|
||||
private val outputType: KType,
|
||||
outputType: KType,
|
||||
private val action: SplitBuilder<T, R>.() -> Unit,
|
||||
) : Action<T, R> {
|
||||
) : AbstractAction<T, R>(outputType) {
|
||||
|
||||
@OptIn(FlowPreview::class)
|
||||
override suspend fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
scope: CoroutineScope?,
|
||||
): DataSet<R> {
|
||||
private fun DataSetBuilder<R>.splitOne(name: Name, data: Data<T>, meta: Meta) {
|
||||
val laminate = Laminate(data.meta, meta)
|
||||
|
||||
suspend fun splitOne(data: NamedData<T>): Flow<NamedData<R>> {
|
||||
val laminate = Laminate(data.meta, meta)
|
||||
|
||||
val split = SplitBuilder<T, R>(data.name, data.meta).apply(action)
|
||||
val split = SplitBuilder<T, R>(name, data.meta).apply(action)
|
||||
|
||||
|
||||
// apply individual fragment rules to result
|
||||
return split.fragments.entries.asFlow().map { (fragmentName, rule) ->
|
||||
val env = SplitBuilder.FragmentRule<T, R>(fragmentName, laminate.toMutableMeta()).apply(rule)
|
||||
//data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
|
||||
@OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) {
|
||||
// apply individual fragment rules to result
|
||||
split.fragments.forEach { (fragmentName, rule) ->
|
||||
val env = SplitBuilder.FragmentRule<T, R>(
|
||||
fragmentName,
|
||||
laminate.toMutableMeta(),
|
||||
outputType
|
||||
).apply(rule)
|
||||
//data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
|
||||
|
||||
data(
|
||||
fragmentName,
|
||||
@Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) {
|
||||
env.result(data.await())
|
||||
}.named(fragmentName)
|
||||
}
|
||||
}
|
||||
|
||||
return ActiveDataTree<R>(outputType) {
|
||||
populate(dataSet.flow().flatMapConcat(transform = ::splitOne))
|
||||
scope?.launch {
|
||||
dataSet.updates.collect { name ->
|
||||
//clear old nodes
|
||||
remove(name)
|
||||
//collect new items
|
||||
populate(dataSet.flowChildren(name).flatMapConcat(transform = ::splitOne))
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
|
||||
data.forEach { splitOne(it.name, it.data, meta) }
|
||||
}
|
||||
|
||||
override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
|
||||
remove(updateKey)
|
||||
dataSet[updateKey]?.let { splitOne(updateKey, it, meta) }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Action that splits each incoming element into a number of fragments defined in builder
|
||||
*/
|
||||
@DFExperimental
|
||||
@Suppress("FunctionName")
|
||||
public inline fun <T : Any, reified R : Any> Action.Companion.split(
|
||||
noinline builder: SplitBuilder<T, R>.() -> Unit,
|
||||
): Action<T, R> = SplitAction(typeOf<R>(), builder)
|
@ -1,118 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
/**
|
||||
* A mutable [DataTree.Companion.active]. It
|
||||
*/
|
||||
public class ActiveDataTree<T : Any>(
|
||||
override val dataType: KType,
|
||||
) : DataTree<T>, DataSetBuilder<T>, ActiveDataSet<T> {
|
||||
private val mutex = Mutex()
|
||||
private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
|
||||
|
||||
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = mutex.withLock {
|
||||
treeItems.filter { !it.key.body.startsWith("@") }
|
||||
}
|
||||
|
||||
private val _updates = MutableSharedFlow<Name>()
|
||||
|
||||
override val updates: Flow<Name>
|
||||
get() = _updates
|
||||
|
||||
private suspend fun remove(token: NameToken) {
|
||||
mutex.withLock {
|
||||
if (treeItems.remove(token) != null) {
|
||||
_updates.emit(token.asName())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun remove(name: Name) {
|
||||
if (name.isEmpty()) error("Can't remove the root node")
|
||||
(getItem(name.cutLast()).tree as? ActiveDataTree)?.remove(name.lastOrNull()!!)
|
||||
}
|
||||
|
||||
private suspend fun set(token: NameToken, data: Data<T>) {
|
||||
mutex.withLock {
|
||||
treeItems[token] = DataTreeItem.Leaf(data)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun getOrCreateNode(token: NameToken): ActiveDataTree<T> =
|
||||
(treeItems[token] as? DataTreeItem.Node<T>)?.tree as? ActiveDataTree<T>
|
||||
?: ActiveDataTree<T>(dataType).also {
|
||||
mutex.withLock {
|
||||
treeItems[token] = DataTreeItem.Node(it)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun getOrCreateNode(name: Name): ActiveDataTree<T> {
|
||||
return when (name.length) {
|
||||
0 -> this
|
||||
1 -> getOrCreateNode(name.firstOrNull()!!)
|
||||
else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, data: Data<T>?) {
|
||||
if (data == null) {
|
||||
remove(name)
|
||||
} else {
|
||||
when (name.length) {
|
||||
0 -> error("Can't add data with empty name")
|
||||
1 -> set(name.firstOrNull()!!, data)
|
||||
2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
|
||||
}
|
||||
}
|
||||
_updates.emit(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy given data set and mirror its changes to this [ActiveDataTree] in [this@setAndObserve]. Returns an update [Job]
|
||||
*/
|
||||
public fun CoroutineScope.setAndObserve(name: Name, dataSet: DataSet<T>): Job = launch {
|
||||
emit(name, dataSet)
|
||||
dataSet.updates.collect { nameInBranch ->
|
||||
emit(name + nameInBranch, dataSet.getData(nameInBranch))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a dynamic tree. Initial data is placed synchronously. Updates are propagated via [updatesScope]
|
||||
*/
|
||||
@Suppress("FunctionName")
|
||||
public suspend fun <T : Any> ActiveDataTree(
|
||||
type: KType,
|
||||
block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): ActiveDataTree<T> {
|
||||
val tree = ActiveDataTree<T>(type)
|
||||
tree.block()
|
||||
return tree
|
||||
}
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend inline fun <reified T : Any> ActiveDataTree(
|
||||
crossinline block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): ActiveDataTree<T> = ActiveDataTree<T>(typeOf<T>()).apply { block() }
|
||||
|
||||
|
||||
public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
|
||||
name: Name,
|
||||
noinline block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): Unit = emit(name, ActiveDataTree(typeOf<T>(), block))
|
||||
|
||||
public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
|
||||
name: String,
|
||||
noinline block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): Unit = emit(name.toName(), ActiveDataTree(typeOf<T>(), block))
|
@ -1,52 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.actions.Action
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
import kotlin.reflect.KType
|
||||
|
||||
/**
|
||||
* Remove all values with keys starting with [name]
|
||||
*/
|
||||
internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
|
||||
val toRemove = keys.filter { it.startsWith(name) }
|
||||
toRemove.forEach(::remove)
|
||||
}
|
||||
|
||||
/**
|
||||
* An action that caches results on-demand and recalculates them on source push
|
||||
*/
|
||||
public abstract class CachingAction<in T : Any, out R : Any>(
|
||||
public val outputType: KType,
|
||||
) : Action<T, R> {
|
||||
|
||||
protected abstract fun CoroutineScope.transform(
|
||||
set: DataSet<T>,
|
||||
meta: Meta,
|
||||
key: Name = Name.EMPTY,
|
||||
): Flow<NamedData<R>>
|
||||
|
||||
override suspend fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
scope: CoroutineScope?,
|
||||
): DataSet<R> = ActiveDataTree<R>(outputType) {
|
||||
coroutineScope {
|
||||
populate(transform(dataSet, meta))
|
||||
}
|
||||
scope?.let {
|
||||
dataSet.updates.collect {
|
||||
//clear old nodes
|
||||
remove(it)
|
||||
//collect new items
|
||||
populate(scope.transform(dataSet, meta, it))
|
||||
//FIXME if the target is data, updates are fired twice
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -48,9 +48,9 @@ public val CoroutineScope.monitor: CoroutineMonitor? get() = coroutineContext.mo
|
||||
public val Job.dependencies: Collection<Job> get() = this[Dependencies]?.values ?: emptyList()
|
||||
|
||||
@DFExperimental
|
||||
public val Job.totalWork: Double get() = dependencies.sumByDouble { totalWork } + (monitor?.totalWork ?: 0.0)
|
||||
public val Job.totalWork: Double get() = dependencies.sumOf { totalWork } + (monitor?.totalWork ?: 0.0)
|
||||
@DFExperimental
|
||||
public val Job.workDone: Double get() = dependencies.sumByDouble { workDone } + (monitor?.workDone ?: 0.0)
|
||||
public val Job.workDone: Double get() = dependencies.sumOf { workDone } + (monitor?.workDone ?: 0.0)
|
||||
@DFExperimental
|
||||
public val Job.status: String get() = monitor?.status ?: ""
|
||||
@DFExperimental
|
||||
|
@ -5,7 +5,7 @@ import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.meta.isEmpty
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
@ -14,8 +14,8 @@ import kotlin.reflect.typeOf
|
||||
/**
|
||||
* A data element characterized by its meta
|
||||
*/
|
||||
@Type(Data.TYPE)
|
||||
public interface Data<out T : Any> : Goal<T>, MetaRepr {
|
||||
@DfId(Data.TYPE)
|
||||
public interface Data<out T> : Goal<T>, MetaRepr {
|
||||
/**
|
||||
* Type marker for the data. The type is known before the calculation takes place so it could be checked.
|
||||
*/
|
||||
@ -49,6 +49,7 @@ public interface Data<out T : Any> : Goal<T>, MetaRepr {
|
||||
/**
|
||||
* An empty data containing only meta
|
||||
*/
|
||||
@OptIn(DelicateCoroutinesApi::class)
|
||||
public fun empty(meta: Meta): Data<Nothing> = object : Data<Nothing> {
|
||||
override val type: KType = TYPE_OF_NOTHING
|
||||
override val meta: Meta = meta
|
||||
@ -72,7 +73,7 @@ private class LazyData<T : Any>(
|
||||
override val type: KType,
|
||||
override val meta: Meta = Meta.EMPTY,
|
||||
additionalContext: CoroutineContext = EmptyCoroutineContext,
|
||||
dependencies: Collection<Data<*>> = emptyList(),
|
||||
dependencies: Collection<Goal<*>> = emptyList(),
|
||||
block: suspend () -> T,
|
||||
) : Data<T>, LazyGoal<T>(additionalContext, dependencies, block)
|
||||
|
||||
@ -82,13 +83,17 @@ public class StaticData<T : Any>(
|
||||
override val meta: Meta = Meta.EMPTY,
|
||||
) : Data<T>, StaticGoal<T>(value)
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public inline fun <reified T : Any> Data(value: T, meta: Meta = Meta.EMPTY): StaticData<T> =
|
||||
StaticData(typeOf<T>(), value, meta)
|
||||
|
||||
@Suppress("FunctionName")
|
||||
@DFInternal
|
||||
public fun <T : Any> Data(
|
||||
type: KType,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
context: CoroutineContext = EmptyCoroutineContext,
|
||||
dependencies: Collection<Data<*>> = emptyList(),
|
||||
dependencies: Collection<Goal<*>> = emptyList(),
|
||||
block: suspend () -> T,
|
||||
): Data<T> = LazyData(type, meta, context, dependencies, block)
|
||||
|
||||
@ -97,6 +102,6 @@ public fun <T : Any> Data(
|
||||
public inline fun <reified T : Any> Data(
|
||||
meta: Meta = Meta.EMPTY,
|
||||
context: CoroutineContext = EmptyCoroutineContext,
|
||||
dependencies: Collection<Data<*>> = emptyList(),
|
||||
dependencies: Collection<Goal<*>> = emptyList(),
|
||||
noinline block: suspend () -> T,
|
||||
): Data<T> = Data(typeOf<T>(), meta, context, dependencies, block)
|
||||
|
@ -1,11 +1,15 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.emptyFlow
|
||||
import kotlinx.coroutines.flow.mapNotNull
|
||||
import space.kscience.dataforge.data.Data.Companion.TYPE_OF_NOTHING
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.set
|
||||
import space.kscience.dataforge.names.*
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.endsWith
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.reflect.KType
|
||||
|
||||
public interface DataSet<out T : Any> {
|
||||
@ -16,24 +20,19 @@ public interface DataSet<out T : Any> {
|
||||
public val dataType: KType
|
||||
|
||||
/**
|
||||
* Traverse this provider or its child. The order is not guaranteed.
|
||||
* [root] points to a root name for traversal. If it is empty, traverse this source, if it points to a [Data],
|
||||
* return flow, that contains single [Data], if it points to a node with children, return children.
|
||||
* Meta-data associated with this node. If no meta is provided, returns [Meta.EMPTY].
|
||||
*/
|
||||
public fun flow(): Flow<NamedData<T>>
|
||||
public val meta: Meta
|
||||
|
||||
/**
|
||||
* Traverse this [DataSet] returning named data instances. The order is not guaranteed.
|
||||
*/
|
||||
public operator fun iterator(): Iterator<NamedData<T>>
|
||||
|
||||
/**
|
||||
* Get data with given name.
|
||||
*/
|
||||
public suspend fun getData(name: Name): Data<T>?
|
||||
|
||||
/**
|
||||
* Get a snapshot of names of children of given node. Empty if node does not exist or is a leaf.
|
||||
*
|
||||
* By default traverses the whole tree. Could be optimized in descendants
|
||||
*/
|
||||
public suspend fun listChildren(prefix: Name = Name.EMPTY): List<Name> =
|
||||
flow().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList()
|
||||
public operator fun get(name: Name): Data<T>?
|
||||
|
||||
public companion object {
|
||||
public val META_KEY: Name = "@meta".asName()
|
||||
@ -43,17 +42,35 @@ public interface DataSet<out T : Any> {
|
||||
*/
|
||||
public val EMPTY: DataSet<Nothing> = object : DataSet<Nothing> {
|
||||
override val dataType: KType = TYPE_OF_NOTHING
|
||||
override val meta: Meta get() = Meta.EMPTY
|
||||
|
||||
private val nothing: Nothing get() = error("this is nothing")
|
||||
override fun iterator(): Iterator<NamedData<Nothing>> = emptySequence<NamedData<Nothing>>().iterator()
|
||||
|
||||
override fun flow(): Flow<NamedData<Nothing>> = emptyFlow()
|
||||
|
||||
override suspend fun getData(name: Name): Data<Nothing>? = null
|
||||
override fun get(name: Name): Data<Nothing>? = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public interface ActiveDataSet<T : Any> : DataSet<T> {
|
||||
public fun <T : Any> DataSet<T>.asSequence(): Sequence<NamedData<T>> = object : Sequence<NamedData<T>> {
|
||||
override fun iterator(): Iterator<NamedData<T>> = this@asSequence.iterator()
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a single [Data] in this [DataSet]. Throw error if it is not single.
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.single(): NamedData<T> = asSequence().single()
|
||||
|
||||
public fun <T : Any> DataSet<T>.asIterable(): Iterable<NamedData<T>> = object : Iterable<NamedData<T>> {
|
||||
override fun iterator(): Iterator<NamedData<T>> = this@asIterable.iterator()
|
||||
}
|
||||
|
||||
public operator fun <T : Any> DataSet<T>.get(name: String): Data<T>? = get(name.parseAsName())
|
||||
|
||||
/**
|
||||
* A [DataSet] with propagated updates.
|
||||
*/
|
||||
public interface DataSource<out T : Any> : DataSet<T>, CoroutineScope {
|
||||
|
||||
/**
|
||||
* A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes.
|
||||
* Those can include new data items and replacement of existing ones. The replaced items could update existing data content
|
||||
@ -61,30 +78,38 @@ public interface ActiveDataSet<T : Any> : DataSet<T> {
|
||||
*
|
||||
*/
|
||||
public val updates: Flow<Name>
|
||||
|
||||
/**
|
||||
* Stop generating updates from this [DataSource]
|
||||
*/
|
||||
public fun close() {
|
||||
coroutineContext[Job]?.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is ActiveDataSet) updates else emptyFlow()
|
||||
|
||||
/**
|
||||
* Flow all data nodes with names starting with [branchName]
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.flowChildren(branchName: Name): Flow<NamedData<T>> = this@flowChildren.flow().filter {
|
||||
it.name.startsWith(branchName)
|
||||
}
|
||||
public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is DataSource) updates else emptyFlow()
|
||||
//
|
||||
///**
|
||||
// * Flow all data nodes with names starting with [branchName]
|
||||
// */
|
||||
//public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T>> =
|
||||
// this@children.asSequence().filter {
|
||||
// it.name.startsWith(branchName)
|
||||
// }
|
||||
|
||||
/**
|
||||
* Start computation for all goals in data node and return a job for the whole node
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch {
|
||||
flow().map {
|
||||
asIterable().map {
|
||||
it.launch(this@launch)
|
||||
}.toList().joinAll()
|
||||
}.joinAll()
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSet<T>.join(): Unit = coroutineScope { startAll(this).join() }
|
||||
public suspend fun <T : Any> DataSet<T>.computeAndJoinAll(): Unit = coroutineScope { startAll(this).join() }
|
||||
|
||||
public suspend fun DataSet<*>.toMeta(): Meta = Meta {
|
||||
flow().collect {
|
||||
public fun DataSet<*>.toMeta(): Meta = Meta {
|
||||
forEach {
|
||||
if (it.name.endsWith(DataSet.META_KEY)) {
|
||||
set(it.name, it.meta)
|
||||
} else {
|
||||
@ -96,4 +121,4 @@ public suspend fun DataSet<*>.toMeta(): Meta = Meta {
|
||||
}
|
||||
}
|
||||
|
||||
public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { getData(it)?.named(it) }
|
||||
public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { get(it)?.named(it) }
|
@ -1,14 +1,11 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.isEmpty
|
||||
import space.kscience.dataforge.names.plus
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.reflect.KType
|
||||
|
||||
public interface DataSetBuilder<in T : Any> {
|
||||
@ -17,129 +14,152 @@ public interface DataSetBuilder<in T : Any> {
|
||||
/**
|
||||
* Remove all data items starting with [name]
|
||||
*/
|
||||
public suspend fun remove(name: Name)
|
||||
public fun remove(name: Name)
|
||||
|
||||
public suspend fun emit(name: Name, data: Data<T>?)
|
||||
public fun data(name: Name, data: Data<T>?)
|
||||
|
||||
/**
|
||||
* Set a current state of given [dataSet] into a branch [name]. Does not propagate updates
|
||||
*/
|
||||
public suspend fun emit(name: Name, dataSet: DataSet<T>) {
|
||||
public fun node(name: Name, dataSet: DataSet<T>) {
|
||||
//remove previous items
|
||||
if (name != Name.EMPTY) {
|
||||
remove(name)
|
||||
}
|
||||
|
||||
//Set new items
|
||||
dataSet.flow().collect {
|
||||
emit(name + it.name, it.data)
|
||||
dataSet.forEach {
|
||||
data(name + it.name, it.data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append data to node
|
||||
* Set meta for the given node
|
||||
*/
|
||||
public suspend infix fun String.put(data: Data<T>): Unit = emit(toName(), data)
|
||||
public fun meta(name: Name, meta: Meta)
|
||||
|
||||
/**
|
||||
* Append node
|
||||
*/
|
||||
public suspend infix fun String.put(dataSet: DataSet<T>): Unit = emit(toName(), dataSet)
|
||||
|
||||
/**
|
||||
* Build and append node
|
||||
*/
|
||||
public suspend infix fun String.put(block: suspend DataSetBuilder<T>.() -> Unit): Unit = emit(toName(), block)
|
||||
}
|
||||
|
||||
private class SubSetBuilder<in T : Any>(
|
||||
/**
|
||||
* Define meta in this [DataSet]
|
||||
*/
|
||||
public fun <T : Any> DataSetBuilder<T>.meta(value: Meta): Unit = meta(Name.EMPTY, value)
|
||||
|
||||
/**
|
||||
* Define meta in this [DataSet]
|
||||
*/
|
||||
public fun <T : Any> DataSetBuilder<T>.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta))
|
||||
|
||||
@PublishedApi
|
||||
internal class SubSetBuilder<in T : Any>(
|
||||
private val parent: DataSetBuilder<T>,
|
||||
private val branch: Name,
|
||||
) : DataSetBuilder<T> {
|
||||
override val dataType: KType get() = parent.dataType
|
||||
|
||||
override suspend fun remove(name: Name) {
|
||||
override fun remove(name: Name) {
|
||||
parent.remove(branch + name)
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, data: Data<T>?) {
|
||||
parent.emit(branch + name, data)
|
||||
override fun data(name: Name, data: Data<T>?) {
|
||||
parent.data(branch + name, data)
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, dataSet: DataSet<T>) {
|
||||
parent.emit(branch + name, dataSet)
|
||||
override fun node(name: Name, dataSet: DataSet<T>) {
|
||||
parent.node(branch + name, dataSet)
|
||||
}
|
||||
|
||||
override fun meta(name: Name, meta: Meta) {
|
||||
parent.meta(branch + name, meta)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: Name, block: suspend DataSetBuilder<T>.() -> Unit) {
|
||||
SubSetBuilder(this, name).apply { block() }
|
||||
public inline fun <T : Any> DataSetBuilder<T>.node(
|
||||
name: Name,
|
||||
crossinline block: DataSetBuilder<T>.() -> Unit,
|
||||
) {
|
||||
if (name.isEmpty()) block() else SubSetBuilder(this, name).block()
|
||||
}
|
||||
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, data: Data<T>) {
|
||||
emit(name.toName(), data)
|
||||
public fun <T : Any> DataSetBuilder<T>.data(name: String, value: Data<T>) {
|
||||
data(Name.parse(name), value)
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, set: DataSet<T>) {
|
||||
this.emit(name.toName(), set)
|
||||
public fun <T : Any> DataSetBuilder<T>.node(name: String, set: DataSet<T>) {
|
||||
node(Name.parse(name), set)
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, block: suspend DataSetBuilder<T>.() -> Unit): Unit =
|
||||
this@emit.emit(name.toName(), block)
|
||||
public inline fun <T : Any> DataSetBuilder<T>.node(
|
||||
name: String,
|
||||
crossinline block: DataSetBuilder<T>.() -> Unit,
|
||||
): Unit = node(Name.parse(name), block)
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(data: NamedData<T>) {
|
||||
emit(data.name, data.data)
|
||||
public fun <T : Any> DataSetBuilder<T>.set(value: NamedData<T>) {
|
||||
data(value.name, value.data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Produce lazy [Data] and emit it into the [DataSetBuilder]
|
||||
*/
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
name: String,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline producer: suspend () -> T,
|
||||
) {
|
||||
val data = Data(meta, block = producer)
|
||||
emit(name, data)
|
||||
data(name, data)
|
||||
}
|
||||
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
name: Name,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline producer: suspend () -> T,
|
||||
) {
|
||||
val data = Data(meta, block = producer)
|
||||
emit(name, data)
|
||||
data(name, data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a static data with the fixed value
|
||||
*/
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit =
|
||||
emit(name, Data.static(data, meta))
|
||||
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit =
|
||||
emit(name, Data.static(data, meta))
|
||||
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
name: String,
|
||||
data: T,
|
||||
metaBuilder: MetaBuilder.() -> Unit,
|
||||
): Unit = emit(name.toName(), Data.static(data, Meta(metaBuilder)))
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): Unit = data(name, Data.static(data, meta))
|
||||
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
name: Name,
|
||||
data: T,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): Unit = data(name, Data.static(data, meta))
|
||||
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
name: String,
|
||||
data: T,
|
||||
mutableMeta: MutableMeta.() -> Unit,
|
||||
): Unit = data(Name.parse(name), Data.static(data, Meta(mutableMeta)))
|
||||
|
||||
/**
|
||||
* Update data with given node data and meta with node meta.
|
||||
*/
|
||||
@DFExperimental
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.populate(tree: DataSet<T>): Unit = coroutineScope {
|
||||
tree.flow().collect {
|
||||
public fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit {
|
||||
tree.forEach {
|
||||
//TODO check if the place is occupied
|
||||
emit(it.name, it.data)
|
||||
data(it.name, it.data)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.populate(flow: Flow<NamedData<T>>) {
|
||||
flow.collect {
|
||||
emit(it.name, it.data)
|
||||
//public fun <T : Any> DataSetBuilder<T>.populateFrom(flow: Flow<NamedData<T>>) {
|
||||
// flow.collect {
|
||||
// data(it.name, it.data)
|
||||
// }
|
||||
//}
|
||||
|
||||
public fun <T : Any> DataSetBuilder<T>.populateFrom(sequence: Sequence<NamedData<T>>) {
|
||||
sequence.forEach {
|
||||
data(it.name, it.data)
|
||||
}
|
||||
}
|
||||
|
@ -1,18 +1,25 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.emitAll
|
||||
import kotlinx.coroutines.flow.flow
|
||||
import kotlinx.coroutines.flow.map
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public sealed class DataTreeItem<out T : Any> {
|
||||
public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>()
|
||||
public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>()
|
||||
|
||||
public abstract val meta: Meta
|
||||
|
||||
public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>() {
|
||||
override val meta: Meta get() = tree.meta
|
||||
}
|
||||
|
||||
public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>() {
|
||||
override val meta: Meta get() = data.meta
|
||||
}
|
||||
}
|
||||
|
||||
public val <T : Any> DataTreeItem<T>.type: KType
|
||||
@ -24,63 +31,79 @@ public val <T : Any> DataTreeItem<T>.type: KType
|
||||
/**
|
||||
* A tree-like [DataSet] grouped into the node. All data inside the node must inherit its type
|
||||
*/
|
||||
@Type(DataTree.TYPE)
|
||||
@DfId(DataTree.TYPE)
|
||||
public interface DataTree<out T : Any> : DataSet<T> {
|
||||
|
||||
/**
|
||||
* Children items of this [DataTree] provided asynchronously
|
||||
* Top-level children items of this [DataTree]
|
||||
*/
|
||||
public suspend fun items(): Map<NameToken, DataTreeItem<T>>
|
||||
public val items: Map<NameToken, DataTreeItem<T>>
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> = flow {
|
||||
items().forEach { (token, childItem: DataTreeItem<T>) ->
|
||||
if(!token.body.startsWith("@")) {
|
||||
override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
items.forEach { (token, childItem: DataTreeItem<T>) ->
|
||||
if (!token.body.startsWith("@")) {
|
||||
when (childItem) {
|
||||
is DataTreeItem.Leaf -> emit(childItem.data.named(token.asName()))
|
||||
is DataTreeItem.Node -> emitAll(childItem.tree.flow().map { it.named(token + it.name) })
|
||||
is DataTreeItem.Leaf -> yield(childItem.data.named(token.asName()))
|
||||
is DataTreeItem.Node -> yieldAll(childItem.tree.asSequence().map { it.named(token + it.name) })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun listChildren(prefix: Name): List<Name> =
|
||||
getItem(prefix).tree?.items()?.keys?.map { prefix + it } ?: emptyList()
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? = when (name.length) {
|
||||
override fun get(name: Name): Data<T>? = when (name.length) {
|
||||
0 -> null
|
||||
1 -> items()[name.firstOrNull()!!].data
|
||||
else -> items()[name.firstOrNull()!!].tree?.getData(name.cutFirst())
|
||||
1 -> items[name.firstOrNull()!!].data
|
||||
else -> items[name.firstOrNull()!!].tree?.get(name.cutFirst())
|
||||
}
|
||||
|
||||
public companion object {
|
||||
public const val TYPE: String = "dataTree"
|
||||
|
||||
/**
|
||||
* A name token used to designate tree node meta
|
||||
*/
|
||||
public val META_ITEM_NAME_TOKEN: NameToken = NameToken("@meta")
|
||||
|
||||
@DFInternal
|
||||
public fun <T : Any> emptyWithType(type: KType, meta: Meta = Meta.EMPTY): DataTree<T> = object : DataTree<T> {
|
||||
override val items: Map<NameToken, DataTreeItem<T>> get() = emptyMap()
|
||||
override val dataType: KType get() = type
|
||||
override val meta: Meta get() = meta
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline fun <reified T : Any> empty(meta: Meta = Meta.EMPTY): DataTree<T> =
|
||||
emptyWithType<T>(typeOf<T>(), meta)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun <T: Any> DataSet<T>.getData(name: String): Data<T>? = getData(name.toName())
|
||||
public fun <T : Any> DataTree<T>.listChildren(prefix: Name): List<Name> =
|
||||
getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
|
||||
|
||||
/**
|
||||
* Get a [DataTreeItem] with given [name] or null if the item does not exist
|
||||
*/
|
||||
public tailrec suspend fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
|
||||
public tailrec fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
|
||||
0 -> DataTreeItem.Node(this)
|
||||
1 -> items()[name.firstOrNull()]
|
||||
else -> items()[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
|
||||
1 -> items[name.firstOrNull()]
|
||||
else -> items[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
|
||||
}
|
||||
|
||||
public val <T : Any> DataTreeItem<T>?.tree: DataTree<T>? get() = (this as? DataTreeItem.Node<T>)?.tree
|
||||
public val <T : Any> DataTreeItem<T>?.data: Data<T>? get() = (this as? DataTreeItem.Leaf<T>)?.data
|
||||
|
||||
/**
|
||||
* Flow of all children including nodes
|
||||
* A [Sequence] of all children including nodes
|
||||
*/
|
||||
public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> = flow {
|
||||
items().forEach { (head, item) ->
|
||||
emit(head.asName() to item)
|
||||
public fun <T : Any> DataTree<T>.traverseItems(): Sequence<Pair<Name, DataTreeItem<T>>> = sequence {
|
||||
items.forEach { (head, item) ->
|
||||
yield(head.asName() to item)
|
||||
if (item is DataTreeItem.Node) {
|
||||
val subSequence = item.tree.itemFlow()
|
||||
val subSequence = item.tree.traverseItems()
|
||||
.map { (name, data) -> (head.asName() + name) to data }
|
||||
emitAll(subSequence)
|
||||
yieldAll(subSequence)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -89,8 +112,8 @@ public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> =
|
||||
* Get a branch of this [DataTree] with a given [branchName].
|
||||
* The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree]
|
||||
*/
|
||||
public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> = object : DataTree<T> {
|
||||
override val dataType: KType get() = this@branch.dataType
|
||||
@OptIn(DFInternal::class)
|
||||
public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> =
|
||||
getItem(branchName)?.tree ?: DataTree.emptyWithType(dataType)
|
||||
|
||||
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = getItem(branchName).tree?.items() ?: emptyMap()
|
||||
}
|
||||
public fun <T : Any> DataTree<T>.branch(branchName: String): DataTree<T> = branch(branchName.parseAsName())
|
||||
|
@ -0,0 +1,127 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.ThreadSafe
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.collections.set
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.coroutineContext
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public interface DataSourceBuilder<T : Any> : DataSetBuilder<T>, DataSource<T> {
|
||||
override val updates: MutableSharedFlow<Name>
|
||||
}
|
||||
|
||||
/**
|
||||
* A mutable [DataTree] that propagates updates
|
||||
*/
|
||||
public class DataTreeBuilder<T : Any>(
|
||||
override val dataType: KType,
|
||||
coroutineContext: CoroutineContext,
|
||||
) : DataTree<T>, DataSourceBuilder<T> {
|
||||
|
||||
override val coroutineContext: CoroutineContext =
|
||||
coroutineContext + Job(coroutineContext[Job]) + GoalExecutionRestriction()
|
||||
|
||||
private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
|
||||
|
||||
override val items: Map<NameToken, DataTreeItem<T>>
|
||||
get() = treeItems.filter { !it.key.body.startsWith("@") }
|
||||
|
||||
override val updates: MutableSharedFlow<Name> = MutableSharedFlow<Name>()
|
||||
|
||||
@ThreadSafe
|
||||
private fun remove(token: NameToken) {
|
||||
if (treeItems.remove(token) != null) {
|
||||
launch {
|
||||
updates.emit(token.asName())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun remove(name: Name) {
|
||||
if (name.isEmpty()) error("Can't remove the root node")
|
||||
(getItem(name.cutLast()).tree as? DataTreeBuilder)?.remove(name.lastOrNull()!!)
|
||||
}
|
||||
|
||||
@ThreadSafe
|
||||
private fun set(token: NameToken, data: Data<T>) {
|
||||
treeItems[token] = DataTreeItem.Leaf(data)
|
||||
}
|
||||
|
||||
@ThreadSafe
|
||||
private fun set(token: NameToken, node: DataTree<T>) {
|
||||
treeItems[token] = DataTreeItem.Node(node)
|
||||
}
|
||||
|
||||
private fun getOrCreateNode(token: NameToken): DataTreeBuilder<T> =
|
||||
(treeItems[token] as? DataTreeItem.Node<T>)?.tree as? DataTreeBuilder<T>
|
||||
?: DataTreeBuilder<T>(dataType, coroutineContext).also { set(token, it) }
|
||||
|
||||
private fun getOrCreateNode(name: Name): DataTreeBuilder<T> = when (name.length) {
|
||||
0 -> this
|
||||
1 -> getOrCreateNode(name.firstOrNull()!!)
|
||||
else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
|
||||
}
|
||||
|
||||
override fun data(name: Name, data: Data<T>?) {
|
||||
if (data == null) {
|
||||
remove(name)
|
||||
} else {
|
||||
when (name.length) {
|
||||
0 -> error("Can't add data with empty name")
|
||||
1 -> set(name.firstOrNull()!!, data)
|
||||
2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
|
||||
}
|
||||
}
|
||||
launch {
|
||||
updates.emit(name)
|
||||
}
|
||||
}
|
||||
|
||||
override fun meta(name: Name, meta: Meta) {
|
||||
val item = getItem(name)
|
||||
if (item is DataTreeItem.Leaf) error("TODO: Can't change meta of existing leaf item.")
|
||||
data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a dynamic [DataSource]. Initial data is placed synchronously.
|
||||
*/
|
||||
@DFInternal
|
||||
@Suppress("FunctionName")
|
||||
public fun <T : Any> DataSource(
|
||||
type: KType,
|
||||
parent: CoroutineScope,
|
||||
block: DataSourceBuilder<T>.() -> Unit,
|
||||
): DataTreeBuilder<T> = DataTreeBuilder<T>(type, parent.coroutineContext).apply(block)
|
||||
|
||||
@Suppress("OPT_IN_USAGE", "FunctionName")
|
||||
public inline fun <reified T : Any> DataSource(
|
||||
parent: CoroutineScope,
|
||||
crossinline block: DataSourceBuilder<T>.() -> Unit,
|
||||
): DataTreeBuilder<T> = DataSource(typeOf<T>(), parent) { block() }
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend inline fun <reified T : Any> DataSource(
|
||||
crossinline block: DataSourceBuilder<T>.() -> Unit = {},
|
||||
): DataTreeBuilder<T> = DataTreeBuilder<T>(typeOf<T>(), coroutineContext).apply { block() }
|
||||
|
||||
public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
|
||||
name: Name,
|
||||
parent: CoroutineScope,
|
||||
noinline block: DataSourceBuilder<T>.() -> Unit,
|
||||
): Unit = node(name, DataSource(parent, block))
|
||||
|
||||
public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
|
||||
name: String,
|
||||
parent: CoroutineScope,
|
||||
noinline block: DataSourceBuilder<T>.() -> Unit,
|
||||
): Unit = node(Name.parse(name), DataSource(parent, block))
|
@ -67,7 +67,7 @@ public open class LazyGoal<T>(
|
||||
* If [GoalExecutionRestriction] is present in the [coroutineScope] context, the call could produce a error a warning
|
||||
* depending on the settings.
|
||||
*/
|
||||
@DFExperimental
|
||||
@OptIn(DFExperimental::class)
|
||||
override fun async(coroutineScope: CoroutineScope): Deferred<T> {
|
||||
val log = coroutineScope.coroutineContext[GoalLogger]
|
||||
// Check if context restricts goal computation
|
||||
|
@ -3,11 +3,25 @@ package space.kscience.dataforge.data
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
|
||||
public enum class GoalExecutionRestrictionPolicy {
|
||||
/**
|
||||
* Allow eager execution
|
||||
*/
|
||||
NONE,
|
||||
|
||||
/**
|
||||
* Give warning on eager execution
|
||||
*/
|
||||
WARNING,
|
||||
|
||||
/**
|
||||
* Throw error on eager execution
|
||||
*/
|
||||
ERROR
|
||||
}
|
||||
|
||||
/**
|
||||
* A special coroutine context key that allows or disallows goal execution during configuration time (eager execution).
|
||||
*/
|
||||
public class GoalExecutionRestriction(
|
||||
public val policy: GoalExecutionRestrictionPolicy = GoalExecutionRestrictionPolicy.ERROR,
|
||||
) : CoroutineContext.Element {
|
||||
|
@ -2,6 +2,9 @@ package space.kscience.dataforge.data
|
||||
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
|
||||
/**
|
||||
* Coroutine context element that provides logging capabilities
|
||||
*/
|
||||
public interface GoalLogger : CoroutineContext.Element {
|
||||
override val key: CoroutineContext.Key<*> get() = GoalLogger
|
||||
|
||||
|
@ -15,14 +15,13 @@
|
||||
*/
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
|
||||
public interface GroupRule {
|
||||
public suspend fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
|
||||
public fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
|
||||
|
||||
public companion object {
|
||||
/**
|
||||
@ -33,30 +32,45 @@ public interface GroupRule {
|
||||
* @param defaultTagValue
|
||||
* @return
|
||||
*/
|
||||
@OptIn(DFInternal::class)
|
||||
public fun byMetaValue(
|
||||
scope: CoroutineScope,
|
||||
key: String,
|
||||
defaultTagValue: String,
|
||||
): GroupRule = object : GroupRule {
|
||||
|
||||
override suspend fun <T : Any> gather(
|
||||
override fun <T : Any> gather(
|
||||
set: DataSet<T>,
|
||||
): Map<String, DataSet<T>> {
|
||||
val map = HashMap<String, ActiveDataTree<T>>()
|
||||
val map = HashMap<String, DataSet<T>>()
|
||||
|
||||
set.flow().collect { data ->
|
||||
val tagValue = data.meta[key]?.string ?: defaultTagValue
|
||||
map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data)
|
||||
}
|
||||
if (set is DataSource) {
|
||||
set.forEach { data ->
|
||||
val tagValue: String = data.meta[key]?.string ?: defaultTagValue
|
||||
(map.getOrPut(tagValue) { DataTreeBuilder(set.dataType, set.coroutineContext) } as DataTreeBuilder<T>)
|
||||
.data(data.name, data.data)
|
||||
|
||||
scope.launch {
|
||||
set.updates.collect { name ->
|
||||
val data = set.getData(name)
|
||||
val tagValue = data?.meta[key].string ?: defaultTagValue
|
||||
map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(name, data)
|
||||
set.launch {
|
||||
set.updates.collect { name ->
|
||||
val dataUpdate = set[name]
|
||||
|
||||
val updateTagValue = dataUpdate?.meta?.get(key)?.string ?: defaultTagValue
|
||||
map.getOrPut(updateTagValue) {
|
||||
DataSource(set.dataType, this) {
|
||||
data(name, dataUpdate)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
set.forEach { data ->
|
||||
val tagValue: String = data.meta[key]?.string ?: defaultTagValue
|
||||
(map.getOrPut(tagValue) { StaticDataTree(set.dataType) } as StaticDataTree<T>)
|
||||
.data(data.name, data.data)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return map
|
||||
}
|
||||
}
|
||||
|
@ -9,6 +9,9 @@ public interface NamedData<out T : Any> : Named, Data<T> {
|
||||
public val data: Data<T>
|
||||
}
|
||||
|
||||
public operator fun NamedData<*>.component1(): Name = name
|
||||
public operator fun <T: Any> NamedData<T>.component2(): Data<T> = data
|
||||
|
||||
private class NamedDataImpl<out T : Any>(
|
||||
override val name: Name,
|
||||
override val data: Data<T>,
|
||||
|
@ -1,7 +1,6 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.reflect.KType
|
||||
@ -12,15 +11,16 @@ internal class StaticDataTree<T : Any>(
|
||||
override val dataType: KType,
|
||||
) : DataSetBuilder<T>, DataTree<T> {
|
||||
|
||||
private val items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
|
||||
private val _items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
|
||||
|
||||
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = items.filter { !it.key.body.startsWith("@") }
|
||||
override val items: Map<NameToken, DataTreeItem<T>>
|
||||
get() = _items.filter { !it.key.body.startsWith("@") }
|
||||
|
||||
override suspend fun remove(name: Name) {
|
||||
override fun remove(name: Name) {
|
||||
when (name.length) {
|
||||
0 -> error("Can't remove root tree node")
|
||||
1 -> items.remove(name.firstOrNull()!!)
|
||||
else -> (items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
|
||||
1 -> _items.remove(name.firstOrNull()!!)
|
||||
else -> (_items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,51 +28,55 @@ internal class StaticDataTree<T : Any>(
|
||||
0 -> this
|
||||
1 -> {
|
||||
val itemName = name.firstOrNull()!!
|
||||
(items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
|
||||
items[itemName] = DataTreeItem.Node(it)
|
||||
(_items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
|
||||
_items[itemName] = DataTreeItem.Node(it)
|
||||
}
|
||||
}
|
||||
else -> getOrCreateNode(name.cutLast()).getOrCreateNode(name.lastOrNull()!!.asName())
|
||||
}
|
||||
|
||||
private suspend fun set(name: Name, item: DataTreeItem<T>?) {
|
||||
private fun set(name: Name, item: DataTreeItem<T>?) {
|
||||
if (name.isEmpty()) error("Can't set top level tree node")
|
||||
if (item == null) {
|
||||
remove(name)
|
||||
} else {
|
||||
getOrCreateNode(name.cutLast()).items[name.lastOrNull()!!] = item
|
||||
getOrCreateNode(name.cutLast())._items[name.lastOrNull()!!] = item
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, data: Data<T>?) {
|
||||
override fun data(name: Name, data: Data<T>?) {
|
||||
set(name, data?.let { DataTreeItem.Leaf(it) })
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, dataSet: DataSet<T>) {
|
||||
override fun node(name: Name, dataSet: DataSet<T>) {
|
||||
if (dataSet is StaticDataTree) {
|
||||
set(name, DataTreeItem.Node(dataSet))
|
||||
} else {
|
||||
coroutineScope {
|
||||
dataSet.flow().collect {
|
||||
emit(name + it.name, it.data)
|
||||
}
|
||||
dataSet.forEach {
|
||||
data(name + it.name, it.data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun meta(name: Name, meta: Meta) {
|
||||
val item = getItem(name)
|
||||
if (item is DataTreeItem.Leaf) TODO("Can't change meta of existing leaf item.")
|
||||
data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
|
||||
}
|
||||
}
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend fun <T : Any> DataTree(
|
||||
public inline fun <T : Any> DataTree(
|
||||
dataType: KType,
|
||||
block: suspend DataSetBuilder<T>.() -> Unit,
|
||||
block: DataSetBuilder<T>.() -> Unit,
|
||||
): DataTree<T> = StaticDataTree<T>(dataType).apply { block() }
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend inline fun <reified T : Any> DataTree(
|
||||
noinline block: suspend DataSetBuilder<T>.() -> Unit,
|
||||
public inline fun <reified T : Any> DataTree(
|
||||
noinline block: DataSetBuilder<T>.() -> Unit,
|
||||
): DataTree<T> = DataTree(typeOf<T>(), block)
|
||||
|
||||
@OptIn(DFExperimental::class)
|
||||
public suspend fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType){
|
||||
populate(this@seal)
|
||||
public fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType) {
|
||||
populateFrom(this@seal)
|
||||
}
|
@ -4,8 +4,11 @@ import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.filter
|
||||
import kotlinx.coroutines.flow.map
|
||||
import kotlinx.coroutines.flow.mapNotNull
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
|
||||
|
||||
@ -13,34 +16,58 @@ import kotlin.reflect.KType
|
||||
* A stateless filtered [DataSet]
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.filter(
|
||||
predicate: suspend (Name, Data<T>) -> Boolean,
|
||||
): ActiveDataSet<T> = object : ActiveDataSet<T> {
|
||||
predicate: (Name, Meta) -> Boolean,
|
||||
): DataSource<T> = object : DataSource<T> {
|
||||
|
||||
override val dataType: KType get() = this@filter.dataType
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> =
|
||||
this@filter.flow().filter { predicate(it.name, it.data) }
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@filter as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? = this@filter.getData(name)?.takeIf {
|
||||
predicate(name, it)
|
||||
|
||||
override val meta: Meta get() = this@filter.meta
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
for (d in this@filter) {
|
||||
if (predicate(d.name, d.meta)) {
|
||||
yield(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
|
||||
predicate(name, it.meta)
|
||||
}
|
||||
|
||||
override val updates: Flow<Name> = this@filter.updates.filter flowFilter@{ name ->
|
||||
val theData = this@filter.getData(name) ?: return@flowFilter false
|
||||
predicate(name, theData)
|
||||
val theData = this@filter[name] ?: return@flowFilter false
|
||||
predicate(name, theData.meta)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a wrapper data set with a given name prefix appended to all names
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) this
|
||||
else object : ActiveDataSet<T> {
|
||||
public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) {
|
||||
this
|
||||
} else object : DataSource<T> {
|
||||
|
||||
override val dataType: KType get() = this@withNamePrefix.dataType
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> = this@withNamePrefix.flow().map { it.data.named(prefix + it.name) }
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@withNamePrefix as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? =
|
||||
name.removeHeadOrNull(name)?.let { this@withNamePrefix.getData(it) }
|
||||
override val meta: Meta get() = this@withNamePrefix.meta
|
||||
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
for (d in this@withNamePrefix) {
|
||||
yield(d.data.named(prefix + d.name))
|
||||
}
|
||||
}
|
||||
|
||||
override fun get(name: Name): Data<T>? =
|
||||
name.removeFirstOrNull(name)?.let { this@withNamePrefix.get(it) }
|
||||
|
||||
override val updates: Flow<Name> get() = this@withNamePrefix.updates.map { prefix + it }
|
||||
}
|
||||
@ -50,22 +77,29 @@ else object : ActiveDataSet<T> {
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branchName.isEmpty()) {
|
||||
this
|
||||
} else object : ActiveDataSet<T> {
|
||||
} else object : DataSource<T> {
|
||||
override val dataType: KType get() = this@branch.dataType
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> = this@branch.flow().mapNotNull {
|
||||
it.name.removeHeadOrNull(branchName)?.let { name ->
|
||||
it.data.named(name)
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@branch as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override val meta: Meta get() = this@branch.meta
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
for (d in this@branch) {
|
||||
d.name.removeFirstOrNull(branchName)?.let { name ->
|
||||
yield(d.data.named(name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? = this@branch.getData(branchName + name)
|
||||
override fun get(name: Name): Data<T>? = this@branch.get(branchName + name)
|
||||
|
||||
override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
|
||||
override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeFirstOrNull(branchName) }
|
||||
}
|
||||
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.toName())
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.parseAsName())
|
||||
|
||||
@DFExperimental
|
||||
public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = getData(Name.EMPTY)
|
||||
public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = get(Name.EMPTY)
|
||||
|
||||
|
@ -1,20 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
|
||||
|
||||
/**
|
||||
* Get a metadata node for this set if it is present
|
||||
*/
|
||||
public suspend fun DataSet<*>.getMeta(): Meta? = getData(DataSet.META_KEY)?.meta
|
||||
|
||||
/**
|
||||
* Add meta-data node to a [DataSet]
|
||||
*/
|
||||
public suspend fun DataSetBuilder<*>.meta(meta: Meta): Unit = emit(DataSet.META_KEY, Data.empty(meta))
|
||||
|
||||
/**
|
||||
* Add meta-data node to a [DataSet]
|
||||
*/
|
||||
public suspend fun DataSetBuilder<*>.meta(metaBuilder: MetaBuilder.() -> Unit): Unit = meta(Meta(metaBuilder))
|
@ -1,18 +1,26 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import kotlin.contracts.InvocationKind
|
||||
import kotlin.contracts.contract
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public data class ValueWithMeta<T>(val meta: Meta, val value: T)
|
||||
|
||||
public suspend fun <T : Any> Data<T>.awaitWithMeta(): ValueWithMeta<T> = ValueWithMeta(meta, await())
|
||||
|
||||
public data class NamedValueWithMeta<T>(val name: Name, val meta: Meta, val value: T)
|
||||
|
||||
public suspend fun <T : Any> NamedData<T>.awaitWithMeta(): NamedValueWithMeta<T> =
|
||||
NamedValueWithMeta(name, meta, await())
|
||||
|
||||
|
||||
/**
|
||||
* Lazily transform this data to another data. By convention [block] should not use external data (be pure).
|
||||
* @param coroutineContext additional [CoroutineContext] elements used for data computation.
|
||||
@ -28,7 +36,7 @@ public inline fun <T : Any, reified R : Any> Data<T>.map(
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine this data with the other data using [block]. See [map] for other details
|
||||
* Combine this data with the other data using [block]. See [Data::map] for other details
|
||||
*/
|
||||
public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
|
||||
other: Data<T2>,
|
||||
@ -48,13 +56,13 @@ public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
|
||||
public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline block: suspend (Collection<T>) -> R,
|
||||
crossinline block: suspend (List<ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
meta,
|
||||
coroutineContext,
|
||||
this
|
||||
) {
|
||||
block(map { it.await() })
|
||||
block(map { it.awaitWithMeta() })
|
||||
}
|
||||
|
||||
@DFInternal
|
||||
@ -62,17 +70,16 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
block: suspend (Map<K, T>) -> R,
|
||||
block: suspend (Map<K, ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
outputType,
|
||||
meta,
|
||||
coroutineContext,
|
||||
this.values
|
||||
) {
|
||||
block(mapValues { it.value.await() })
|
||||
block(mapValues { it.value.awaitWithMeta() })
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Lazily reduce a [Map] of [Data] with any static key.
|
||||
* @param K type of the map key
|
||||
@ -82,56 +89,91 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
|
||||
public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline block: suspend (Map<K, T>) -> R,
|
||||
crossinline block: suspend (Map<K, ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
meta,
|
||||
coroutineContext,
|
||||
this.values
|
||||
) {
|
||||
block(mapValues { it.value.await() })
|
||||
block(mapValues { it.value.awaitWithMeta() })
|
||||
}
|
||||
|
||||
//flow operations
|
||||
//Iterable operations
|
||||
|
||||
/**
|
||||
* Transform a [Flow] of [NamedData] to a single [Data].
|
||||
*/
|
||||
@DFInternal
|
||||
public suspend fun <T : Any, R : Any> Flow<NamedData<T>>.reduceToData(
|
||||
public inline fun <T : Any, R : Any> Iterable<Data<T>>.reduceToData(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
transformation: suspend (Flow<NamedData<T>>) -> R,
|
||||
crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
outputType,
|
||||
meta,
|
||||
coroutineContext,
|
||||
toList()
|
||||
) {
|
||||
transformation(this)
|
||||
transformation(map { it.awaitWithMeta() })
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.reduceToData(
|
||||
public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline transformation: suspend (Flow<NamedData<T>>) -> R,
|
||||
crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = reduceToData(typeOf<R>(), coroutineContext, meta) {
|
||||
transformation(it)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a flow of named data into a single [Data]
|
||||
*/
|
||||
public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.foldToData(
|
||||
public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.foldToData(
|
||||
initial: R,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline block: suspend (result: R, data: NamedData<T>) -> R,
|
||||
crossinline block: suspend (result: R, data: ValueWithMeta<T>) -> R,
|
||||
): Data<R> = reduceToData(
|
||||
coroutineContext, meta
|
||||
) {
|
||||
it.fold(initial, block)
|
||||
it.fold(initial) { acc, t -> block(acc, t) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform an [Iterable] of [NamedData] to a single [Data].
|
||||
*/
|
||||
@DFInternal
|
||||
public inline fun <T : Any, R : Any> Iterable<NamedData<T>>.reduceNamedToData(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
outputType,
|
||||
meta,
|
||||
coroutineContext,
|
||||
toList()
|
||||
) {
|
||||
transformation(map { it.awaitWithMeta() })
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.reduceNamedToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
|
||||
): Data<R> = reduceNamedToData(typeOf<R>(), coroutineContext, meta) {
|
||||
transformation(it)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a [Iterable] of named data into a single [Data]
|
||||
*/
|
||||
public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.foldNamedToData(
|
||||
initial: R,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
|
||||
): Data<R> = reduceNamedToData(
|
||||
coroutineContext, meta
|
||||
) {
|
||||
it.fold(initial) { acc, t -> block(acc, t) }
|
||||
}
|
||||
|
||||
//DataSet operations
|
||||
@ -140,42 +182,40 @@ public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.foldToDa
|
||||
public suspend fun <T : Any, R : Any> DataSet<T>.map(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
metaTransform: MetaBuilder.() -> Unit = {},
|
||||
block: suspend (T) -> R,
|
||||
metaTransform: MutableMeta.() -> Unit = {},
|
||||
block: suspend (NamedValueWithMeta<T>) -> R,
|
||||
): DataTree<R> = DataTree<R>(outputType) {
|
||||
populate(
|
||||
flow().map {
|
||||
val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
|
||||
Data(outputType, newMeta, coroutineContext, listOf(it)) {
|
||||
block(it.await())
|
||||
}.named(it.name)
|
||||
forEach {
|
||||
val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
|
||||
val d = Data(outputType, newMeta, coroutineContext, listOf(it)) {
|
||||
block(it.awaitWithMeta())
|
||||
}
|
||||
)
|
||||
data(it.name, d)
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
noinline metaTransform: MetaBuilder.() -> Unit = {},
|
||||
noinline block: suspend (T) -> R,
|
||||
noinline metaTransform: MutableMeta.() -> Unit = {},
|
||||
noinline block: suspend (NamedValueWithMeta<T>) -> R,
|
||||
): DataTree<R> = map(typeOf<R>(), coroutineContext, metaTransform, block)
|
||||
|
||||
public suspend fun <T : Any> DataSet<T>.forEach(block: suspend (NamedData<T>) -> Unit) {
|
||||
contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
|
||||
flow().collect {
|
||||
block(it)
|
||||
public inline fun <T : Any> DataSet<T>.forEach(block: (NamedData<T>) -> Unit) {
|
||||
for (d in this) {
|
||||
block(d)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
|
||||
public inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline transformation: suspend (Flow<NamedData<T>>) -> R,
|
||||
): Data<R> = flow().reduceToData(coroutineContext, meta, transformation)
|
||||
crossinline transformation: suspend (Iterable<NamedValueWithMeta<T>>) -> R,
|
||||
): Data<R> = asIterable().reduceNamedToData(coroutineContext, meta, transformation)
|
||||
|
||||
public suspend inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
|
||||
public inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
|
||||
initial: R,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline block: suspend (result: R, data: NamedData<T>) -> R,
|
||||
): Data<R> = flow().foldToData(initial, coroutineContext, meta, block)
|
||||
crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
|
||||
): Data<R> = asIterable().foldNamedToData(initial, coroutineContext, meta, block)
|
@ -0,0 +1,2 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
@ -0,0 +1,85 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.filter
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.full.isSubtypeOf
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
|
||||
/**
|
||||
* Cast the node to given type if the cast is possible or return null
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
|
||||
if (!this.type.isSubtypeOf(type)) {
|
||||
null
|
||||
} else {
|
||||
object : Data<R> by (this as Data<R>) {
|
||||
override val type: KType = type
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Select all data matching given type and filters. Does not modify paths
|
||||
*
|
||||
* @param predicate addition filtering condition based on item name and meta. By default, accepts all
|
||||
*/
|
||||
@OptIn(DFExperimental::class)
|
||||
public fun <R : Any> DataSet<*>.filterByType(
|
||||
type: KType,
|
||||
predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
|
||||
): DataSource<R> = object : DataSource<R> {
|
||||
override val dataType = type
|
||||
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@filterByType as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override val meta: Meta get() = this@filterByType.meta
|
||||
|
||||
private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type)
|
||||
&& predicate(name, datum.meta)
|
||||
|
||||
override fun iterator(): Iterator<NamedData<R>> = iterator {
|
||||
for(d in this@filterByType){
|
||||
if(checkDatum(d.name,d.data)){
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
yield(d as NamedData<R>)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun get(name: Name): Data<R>? = this@filterByType[name]?.let { datum ->
|
||||
if (checkDatum(name, datum)) datum.castOrNull(type) else null
|
||||
}
|
||||
|
||||
override val updates: Flow<Name> = this@filterByType.updates.filter { name ->
|
||||
get(name)?.let { datum ->
|
||||
checkDatum(name, datum)
|
||||
} ?: false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Select a single datum of the appropriate type
|
||||
*/
|
||||
public inline fun <reified R : Any> DataSet<*>.filterByType(
|
||||
noinline predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
|
||||
): DataSet<R> = filterByType(typeOf<R>(), predicate)
|
||||
|
||||
/**
|
||||
* Select a single datum if it is present and of given [type]
|
||||
*/
|
||||
public fun <R : Any> DataSet<*>.getByType(type: KType, name: Name): NamedData<R>? =
|
||||
get(name)?.castOrNull<R>(type)?.named(name)
|
||||
|
||||
public inline fun <reified R : Any> DataSet<*>.getByType(name: Name): NamedData<R>? =
|
||||
this@getByType.getByType(typeOf<R>(), name)
|
||||
|
||||
public inline fun <reified R : Any> DataSet<*>.getByType(name: String): NamedData<R>? =
|
||||
this@getByType.getByType(typeOf<R>(), Name.parse(name))
|
@ -0,0 +1,40 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
|
||||
|
||||
/**
|
||||
* Append data to node
|
||||
*/
|
||||
context(DataSetBuilder<T>) public infix fun <T : Any> String.put(data: Data<T>): Unit =
|
||||
data(Name.parse(this), data)
|
||||
|
||||
/**
|
||||
* Append node
|
||||
*/
|
||||
context(DataSetBuilder<T>) public infix fun <T : Any> String.put(dataSet: DataSet<T>): Unit =
|
||||
node(Name.parse(this), dataSet)
|
||||
|
||||
/**
|
||||
* Build and append node
|
||||
*/
|
||||
context(DataSetBuilder<T>) public infix fun <T : Any> String.put(
|
||||
block: DataSetBuilder<T>.() -> Unit,
|
||||
): Unit = node(Name.parse(this), block)
|
||||
|
||||
/**
|
||||
* Copy given data set and mirror its changes to this [DataTreeBuilder] in [this@setAndObserve]. Returns an update [Job]
|
||||
*/
|
||||
context(DataSetBuilder<T>) public fun <T : Any> CoroutineScope.setAndWatch(
|
||||
name: Name,
|
||||
dataSet: DataSet<T>,
|
||||
): Job = launch {
|
||||
node(name, dataSet)
|
||||
dataSet.updates.collect { nameInBranch ->
|
||||
data(name + nameInBranch, dataSet.get(nameInBranch))
|
||||
}
|
||||
}
|
@ -1,64 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.filter
|
||||
import kotlinx.coroutines.flow.map
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.matches
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.full.isSubtypeOf
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
|
||||
/**
|
||||
* Cast the node to given type if the cast is possible or return null
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
|
||||
if (!this.type.isSubtypeOf(type)) null else object : Data<R> by (this as Data<R>) {
|
||||
override val type: KType = type
|
||||
}
|
||||
|
||||
/**
|
||||
* Select all data matching given type and filters. Does not modify paths
|
||||
*/
|
||||
@OptIn(DFExperimental::class)
|
||||
@PublishedApi
|
||||
internal fun <R : Any> DataSet<*>.select(
|
||||
type: KType,
|
||||
namePattern: Name? = null,
|
||||
): ActiveDataSet<R> = object : ActiveDataSet<R> {
|
||||
override val dataType = type
|
||||
|
||||
|
||||
override fun flow(): Flow<NamedData<R>> = this@select.flow().filter { datum ->
|
||||
datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern))
|
||||
}.map {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
it as NamedData<R>
|
||||
}
|
||||
|
||||
override suspend fun getData(name: Name): Data<R>? = this@select.getData(name)?.castOrNull(type)
|
||||
|
||||
override val updates: Flow<Name> = this@select.updates.filter {
|
||||
val datum = this@select.getData(it)
|
||||
datum?.type?.isSubtypeOf(type) ?: false
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Select a single datum of the appropriate type
|
||||
*/
|
||||
public inline fun <reified R : Any> DataSet<*>.select(namePattern: Name? = null): DataSet<R> =
|
||||
select(typeOf<R>(), namePattern)
|
||||
|
||||
public suspend fun <R : Any> DataSet<*>.selectOne(type: KType, name: Name): NamedData<R>? =
|
||||
getData(name)?.castOrNull<R>(type)?.named(name)
|
||||
|
||||
public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: Name): NamedData<R>? = selectOne(typeOf<R>(), name)
|
||||
|
||||
public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: String): NamedData<R>? =
|
||||
selectOne(typeOf<R>(), name.toName())
|
@ -1,42 +1,50 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import org.junit.jupiter.api.Test
|
||||
import space.kscience.dataforge.actions.Action
|
||||
import space.kscience.dataforge.actions.invoke
|
||||
import space.kscience.dataforge.actions.map
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@Suppress("EXPERIMENTAL_API_USAGE")
|
||||
class ActionsTest {
|
||||
val data: DataTree<Int> = runBlocking {
|
||||
DataTree {
|
||||
@OptIn(DFExperimental::class, ExperimentalCoroutinesApi::class)
|
||||
internal class ActionsTest {
|
||||
@Test
|
||||
fun testStaticMapAction() = runTest {
|
||||
val data: DataTree<Int> = DataTree {
|
||||
repeat(10) {
|
||||
static(it.toString(), it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testStaticMapAction() {
|
||||
val plusOne = Action.map<Int, Int> {
|
||||
result { it + 1 }
|
||||
}
|
||||
runBlocking {
|
||||
val result = plusOne.execute(data)
|
||||
assertEquals(2, result.getData("1")?.await())
|
||||
}
|
||||
val result = plusOne(data)
|
||||
assertEquals(2, result["1"]?.await())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testDynamicMapAction() {
|
||||
fun testDynamicMapAction() = runTest {
|
||||
val data: DataSourceBuilder<Int> = DataSource()
|
||||
|
||||
val plusOne = Action.map<Int, Int> {
|
||||
result { it + 1 }
|
||||
}
|
||||
val datum = runBlocking {
|
||||
val result = plusOne.execute(data, scope = this)
|
||||
result.getData("1")?.await()
|
||||
|
||||
val result = plusOne(data)
|
||||
|
||||
repeat(10) {
|
||||
data.static(it.toString(), it)
|
||||
}
|
||||
assertEquals(2, datum)
|
||||
|
||||
delay(20)
|
||||
|
||||
assertEquals(2, result["1"]?.await())
|
||||
data.close()
|
||||
}
|
||||
|
||||
}
|
@ -1,9 +1,8 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.asName
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -20,10 +19,10 @@ internal class DataTreeBuilderTest {
|
||||
static("c.f", "c.f")
|
||||
}
|
||||
runBlocking {
|
||||
assertEquals("a", node.getData("primary.a")?.await())
|
||||
assertEquals("b", node.getData("primary.b")?.await())
|
||||
assertEquals("c.d", node.getData("c.d")?.await())
|
||||
assertEquals("c.f", node.getData("c.f")?.await())
|
||||
assertEquals("a", node["primary.a"]?.await())
|
||||
assertEquals("b", node["primary.b"]?.await())
|
||||
assertEquals("c.d", node["c.d"]?.await())
|
||||
assertEquals("c.f", node["c.f"]?.await())
|
||||
}
|
||||
}
|
||||
|
||||
@ -43,12 +42,12 @@ internal class DataTreeBuilderTest {
|
||||
static("b", "b")
|
||||
}
|
||||
static("root", "root")
|
||||
populate(updateData)
|
||||
populateFrom(updateData)
|
||||
}
|
||||
|
||||
runBlocking {
|
||||
assertEquals("a", node.getData("update.a")?.await())
|
||||
assertEquals("a", node.getData("primary.a")?.await())
|
||||
assertEquals("a", node["update.a"]?.await())
|
||||
assertEquals("a", node["primary.a"]?.await())
|
||||
}
|
||||
}
|
||||
|
||||
@ -57,7 +56,7 @@ internal class DataTreeBuilderTest {
|
||||
try {
|
||||
lateinit var updateJob: Job
|
||||
supervisorScope {
|
||||
val subNode = ActiveDataTree<Int> {
|
||||
val subNode = DataSource<Int> {
|
||||
updateJob = launch {
|
||||
repeat(10) {
|
||||
delay(10)
|
||||
@ -71,8 +70,8 @@ internal class DataTreeBuilderTest {
|
||||
println(it)
|
||||
}
|
||||
}
|
||||
val rootNode = ActiveDataTree<Int> {
|
||||
setAndObserve("sub".toName(), subNode)
|
||||
val rootNode = DataSource<Int> {
|
||||
setAndWatch("sub".asName(), subNode)
|
||||
}
|
||||
|
||||
launch {
|
||||
@ -81,11 +80,11 @@ internal class DataTreeBuilderTest {
|
||||
}
|
||||
}
|
||||
updateJob.join()
|
||||
assertEquals(9, rootNode.getData("sub.value")?.await())
|
||||
assertEquals(9, rootNode["sub.value"]?.await())
|
||||
cancel()
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
if (t !is CancellationException) throw t
|
||||
if (t !is CancellationException) throw t
|
||||
}
|
||||
|
||||
}
|
||||
|
23
dataforge-io/README.md
Normal file
23
dataforge-io/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-io
|
||||
|
||||
IO module
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-io:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-io:0.7.0")
|
||||
}
|
||||
```
|
@ -1,29 +1,26 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
description = "IO module"
|
||||
|
||||
val ioVersion = "0.2.1"
|
||||
|
||||
kscience {
|
||||
useSerialization(sourceSet = ru.mipt.npm.gradle.DependencySourceSet.TEST) {
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
useSerialization()
|
||||
useSerialization(sourceSet = space.kscience.gradle.DependencySourceSet.TEST) {
|
||||
cbor()
|
||||
}
|
||||
}
|
||||
|
||||
//val ioVersion by rootProject.extra("0.2.0-npm-dev-11")
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
commonMain {
|
||||
dependencies {
|
||||
api(project(":dataforge-context"))
|
||||
api("io.ktor:ktor-io:${ru.mipt.npm.gradle.KScienceVersions.ktorVersion}")
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
api(projects.dataforgeContext)
|
||||
api("org.jetbrains.kotlinx:kotlinx-io-core:$ioVersion")
|
||||
api("org.jetbrains.kotlinx:kotlinx-io-bytestring:$ioVersion")
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE
|
||||
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
|
||||
}
|
23
dataforge-io/dataforge-io-yaml/README.md
Normal file
23
dataforge-io/dataforge-io-yaml/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-io-yaml
|
||||
|
||||
YAML meta IO
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-io-yaml:0.7.0")
|
||||
}
|
||||
```
|
@ -1,32 +1,23 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
// id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
description = "YAML meta IO"
|
||||
|
||||
kscience {
|
||||
useSerialization{
|
||||
yamlKt("0.9.0-dev-1")
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
dependencies {
|
||||
api(projects.dataforgeIo)
|
||||
}
|
||||
}
|
||||
|
||||
repositories{
|
||||
maven("https://dl.bintray.com/mamoe/yamlkt")
|
||||
}
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
commonMain{
|
||||
dependencies {
|
||||
api(project(":dataforge-io"))
|
||||
}
|
||||
}
|
||||
useSerialization{
|
||||
yamlKt()
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE
|
||||
maturity = space.kscience.gradle.Maturity.PROTOTYPE
|
||||
description ="""
|
||||
YAML meta converters and Front Matter envelope format
|
||||
""".trimIndent()
|
||||
|
@ -1,123 +1,97 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import io.ktor.utils.io.core.readBytes
|
||||
import io.ktor.utils.io.core.readUTF8Line
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import kotlinx.io.readByteString
|
||||
import kotlinx.io.writeString
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.io.*
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
|
||||
@DFExperimental
|
||||
public class FrontMatterEnvelopeFormat(
|
||||
private val io: IOPlugin,
|
||||
private val meta: Meta = Meta.EMPTY,
|
||||
private val metaFormatFactory: MetaFormatFactory = YamlMetaFormat,
|
||||
) : EnvelopeFormat {
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope {
|
||||
var line: String
|
||||
var offset = 0u
|
||||
do {
|
||||
line = input.readUTF8Line() ?: error("Input does not contain front matter separator")
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
override fun readFrom(binary: Binary): Envelope = binary.read {
|
||||
var offset = 0
|
||||
|
||||
val readMetaFormat =
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
||||
?.let { io.resolveMetaFormat(it) } ?: space.kscience.dataforge.io.yaml.YamlMetaFormat
|
||||
offset += discardWithSeparator(
|
||||
SEPARATOR,
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
//TODO replace by preview
|
||||
val meta = Binary {
|
||||
do {
|
||||
line = input.readSafeUtf8Line()
|
||||
writeUtf8String(line + "\r\n")
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
}.read {
|
||||
readMetaFormat.readMeta(input)
|
||||
val line = ByteArray {
|
||||
offset += readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
}.decodeToString()
|
||||
|
||||
val readMetaFormat = line.trim().takeIf { it.isNotBlank() }?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
|
||||
|
||||
val packet = ByteArray {
|
||||
offset += readWithSeparatorTo(this, SEPARATOR)
|
||||
}
|
||||
return PartialEnvelope(meta, offset, null)
|
||||
|
||||
offset += discardLine()
|
||||
|
||||
val meta = readMetaFormat.readFrom(packet.asBinary())
|
||||
Envelope(meta, binary.view(offset))
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope {
|
||||
var line: String
|
||||
do {
|
||||
line = input.readSafeUtf8Line() //?: error("Input does not contain front matter separator")
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
override fun readFrom(source: Source): Envelope = readFrom(source.readBinary())
|
||||
|
||||
val readMetaFormat =
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
||||
?.let { io.resolveMetaFormat(it) } ?: space.kscience.dataforge.io.yaml.YamlMetaFormat
|
||||
|
||||
val meta = Binary {
|
||||
do {
|
||||
writeUtf8String(input.readSafeUtf8Line() + "\r\n")
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
}.read {
|
||||
readMetaFormat.readMeta(input)
|
||||
}
|
||||
val bytes = input.readBytes()
|
||||
val data = bytes.asBinary()
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory(formatMeta, this@FrontMatterEnvelopeFormat.io.context)
|
||||
output.writeRawString("${space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
|
||||
metaFormat.run { this.writeObject(output, envelope.meta) }
|
||||
output.writeRawString("${space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
|
||||
val metaFormat = metaFormatFactory.build(io.context, meta)
|
||||
val formatSuffix = if (metaFormat is YamlMetaFormat) "" else metaFormatFactory.shortName
|
||||
sink.writeString("$SEPARATOR${formatSuffix}\r\n")
|
||||
metaFormat.run { metaFormat.writeTo(sink, obj.meta) }
|
||||
sink.writeString("$SEPARATOR\r\n")
|
||||
//Printing data
|
||||
envelope.data?.let { data ->
|
||||
output.writeBinary(data)
|
||||
obj.data?.let { data ->
|
||||
sink.writeBinary(data)
|
||||
}
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
META_KEY put meta
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
public const val SEPARATOR: String = "---"
|
||||
public val SEPARATOR: ByteString = "---".encodeToByteString()
|
||||
|
||||
private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
||||
return space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat(context.io, meta)
|
||||
override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "frontMatter"
|
||||
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat {
|
||||
return FrontMatterEnvelopeFormat(context.io, meta)
|
||||
}
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = binary.read {
|
||||
val line = readSafeUtf8Line()
|
||||
return@read if (line.startsWith("---")) {
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.invoke()
|
||||
//read raw string to avoid UTF issues
|
||||
val line = readByteString(3)
|
||||
return@read if (line == "---".encodeToByteString()) {
|
||||
default
|
||||
} else {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private val default by lazy { space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.invoke() }
|
||||
private val default by lazy { build(Global, Meta.EMPTY) }
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope =
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.readPartial(input)
|
||||
override fun readFrom(binary: Binary): Envelope = default.readFrom(binary)
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
): Unit = space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta)
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
): Unit = default.writeTo(sink, obj)
|
||||
|
||||
|
||||
override fun readObject(input: Input): Envelope = space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.readObject(input)
|
||||
override fun readFrom(source: Source): Envelope = default.readFrom(source)
|
||||
|
||||
}
|
||||
}
|
@ -1,82 +1,84 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readString
|
||||
import kotlinx.io.writeString
|
||||
import net.mamoe.yamlkt.*
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.io.MetaFormat
|
||||
import space.kscience.dataforge.io.MetaFormatFactory
|
||||
import space.kscience.dataforge.io.readUtf8String
|
||||
import space.kscience.dataforge.io.writeUtf8String
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.descriptors.ItemDescriptor
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.descriptors.get
|
||||
import space.kscience.dataforge.names.NameToken
|
||||
import space.kscience.dataforge.names.withIndex
|
||||
import space.kscience.dataforge.values.ListValue
|
||||
import space.kscience.dataforge.values.Null
|
||||
import space.kscience.dataforge.values.parseValue
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.collections.set
|
||||
|
||||
public fun Meta.toYaml(): YamlMap {
|
||||
val map: Map<String, Any?> = items.entries.associate { (key, item) ->
|
||||
key.toString() to when (item) {
|
||||
is MetaItemValue -> {
|
||||
item.value.value
|
||||
}
|
||||
is MetaItemNode -> {
|
||||
item.node.toYaml()
|
||||
}
|
||||
key.toString() to if (item.isLeaf) {
|
||||
item.value?.value
|
||||
} else {
|
||||
item.toYaml()
|
||||
}
|
||||
}
|
||||
|
||||
return YamlMap(map)
|
||||
}
|
||||
|
||||
private class YamlMeta(private val yamlMap: YamlMap, private val descriptor: NodeDescriptor? = null) : MetaBase() {
|
||||
private class YamlMeta(private val yamlMap: YamlMap, private val descriptor: MetaDescriptor? = null) : Meta {
|
||||
|
||||
private fun buildItems(): Map<NameToken, MetaItem> {
|
||||
val map = LinkedHashMap<NameToken, MetaItem>()
|
||||
override val value: Value?
|
||||
get() = yamlMap.getStringOrNull(null)?.let { Value.parse(it) }
|
||||
|
||||
private fun buildItems(): Map<NameToken, Meta> {
|
||||
val map = LinkedHashMap<NameToken, Meta>()
|
||||
|
||||
yamlMap.content.entries.forEach { (key, value) ->
|
||||
val stringKey = key.toString()
|
||||
val itemDescriptor = descriptor?.items?.get(stringKey)
|
||||
val itemDescriptor = descriptor?.get(stringKey)
|
||||
val token = NameToken(stringKey)
|
||||
when (value) {
|
||||
YamlNull -> Null.asMetaItem()
|
||||
is YamlLiteral -> map[token] = value.content.parseValue().asMetaItem()
|
||||
is YamlMap -> map[token] = value.toMeta().asMetaItem()
|
||||
YamlNull -> Meta(Null)
|
||||
is YamlLiteral -> map[token] = Meta(Value.parse(value.content))
|
||||
is YamlMap -> map[token] = value.toMeta()
|
||||
is YamlList -> if (value.all { it is YamlLiteral }) {
|
||||
val listValue = ListValue(
|
||||
value.map {
|
||||
//We already checked that all values are primitives
|
||||
(it as YamlLiteral).content.parseValue()
|
||||
Value.parse((it as YamlLiteral).content)
|
||||
}
|
||||
)
|
||||
map[token] = MetaItemValue(listValue)
|
||||
map[token] = Meta(listValue)
|
||||
} else value.forEachIndexed { index, yamlElement ->
|
||||
val indexKey = (itemDescriptor as? NodeDescriptor)?.indexKey ?: ItemDescriptor.DEFAULT_INDEX_KEY
|
||||
val indexKey = itemDescriptor?.indexKey
|
||||
val indexValue: String = (yamlElement as? YamlMap)?.getStringOrNull(indexKey)
|
||||
?: index.toString() //In case index is non-string, the backward transformation will be broken.
|
||||
|
||||
val tokenWithIndex = token.withIndex(indexValue)
|
||||
map[tokenWithIndex] = yamlElement.toMetaItem(itemDescriptor)
|
||||
map[tokenWithIndex] = yamlElement.toMeta(itemDescriptor)
|
||||
}
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
override val items: Map<NameToken, MetaItem> get() = buildItems()
|
||||
override val items: Map<NameToken, Meta> get() = buildItems()
|
||||
|
||||
override fun toString(): String = Meta.toString(this)
|
||||
override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta)
|
||||
override fun hashCode(): Int = Meta.hashCode(this)
|
||||
}
|
||||
|
||||
public fun YamlElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem = when (this) {
|
||||
YamlNull -> Null.asMetaItem()
|
||||
is YamlLiteral -> content.parseValue().asMetaItem()
|
||||
is YamlMap -> toMeta().asMetaItem()
|
||||
public fun YamlElement.toMeta(descriptor: MetaDescriptor? = null): Meta = when (this) {
|
||||
YamlNull -> Meta(Null)
|
||||
is YamlLiteral -> Meta(Value.parse(content))
|
||||
is YamlMap -> toMeta()
|
||||
//We can't return multiple items therefore we create top level node
|
||||
is YamlList -> YamlMap(mapOf("@yamlArray" to this)).toMetaItem(descriptor)
|
||||
is YamlList -> YamlMap(mapOf("@yamlArray" to this)).toMeta(descriptor)
|
||||
}
|
||||
|
||||
public fun YamlMap.toMeta(): Meta = YamlMeta(this)
|
||||
@ -85,38 +87,32 @@ public fun YamlMap.toMeta(): Meta = YamlMeta(this)
|
||||
/**
|
||||
* Represent meta as Yaml
|
||||
*/
|
||||
@DFExperimental
|
||||
public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) {
|
||||
val yaml = meta.toYaml()
|
||||
val string = Yaml.encodeToString(yaml)
|
||||
output.writeUtf8String(string)
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
val yaml: YamlMap = meta.toYaml()
|
||||
val string = Yaml.encodeToString(YamlMap.serializer(), yaml)
|
||||
sink.writeString(string)
|
||||
}
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
|
||||
val yaml = Yaml.decodeYamlMapFromString(input.readUtf8String())
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta {
|
||||
val yaml = Yaml.decodeYamlMapFromString(source.readString())
|
||||
return yaml.toMeta()
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.name.toString()
|
||||
META_KEY put meta
|
||||
}
|
||||
|
||||
public companion object : MetaFormatFactory {
|
||||
override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
|
||||
override fun build(context: Context, meta: Meta): MetaFormat = YamlMetaFormat(meta)
|
||||
|
||||
override val shortName: String = "yaml"
|
||||
|
||||
override val key: Short = 0x594d //YM
|
||||
|
||||
private val default = YamlMetaFormat()
|
||||
private val default = YamlMetaFormat(Meta.EMPTY)
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit =
|
||||
default.writeMeta(output, meta, descriptor)
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.writeMeta(sink, meta, descriptor)
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta =
|
||||
default.readMeta(input, descriptor)
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
default.readMeta(source, descriptor)
|
||||
}
|
||||
}
|
@ -0,0 +1,30 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import space.kscience.dataforge.context.AbstractPlugin
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.PluginFactory
|
||||
import space.kscience.dataforge.context.PluginTag
|
||||
import space.kscience.dataforge.io.EnvelopeFormatFactory
|
||||
import space.kscience.dataforge.io.IOPlugin
|
||||
import space.kscience.dataforge.io.MetaFormatFactory
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
|
||||
public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
public val io: IOPlugin by require(IOPlugin)
|
||||
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
override fun content(target: String): Map<Name, Any> = when (target) {
|
||||
MetaFormatFactory.META_FORMAT_TYPE -> mapOf("yaml".asName() to YamlMetaFormat)
|
||||
EnvelopeFormatFactory.ENVELOPE_FORMAT_TYPE -> mapOf(FrontMatterEnvelopeFormat.name to FrontMatterEnvelopeFormat)
|
||||
else -> super.content(target)
|
||||
}
|
||||
|
||||
public companion object : PluginFactory<YamlPlugin> {
|
||||
override val tag: PluginTag = PluginTag("io.yaml", group = PluginTag.DATAFORGE_GROUP)
|
||||
|
||||
override fun build(context: Context, meta: Meta): YamlPlugin = YamlPlugin(meta)
|
||||
}
|
||||
}
|
@ -0,0 +1,37 @@
|
||||
@file:OptIn(DFExperimental::class)
|
||||
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.io
|
||||
import space.kscience.dataforge.io.readEnvelope
|
||||
import space.kscience.dataforge.io.toByteArray
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
internal class FrontMatterEnvelopeFormatTest {
|
||||
|
||||
val context = Context {
|
||||
plugin(YamlPlugin)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun frontMatter(){
|
||||
val text = """
|
||||
---
|
||||
content_type: magprog
|
||||
magprog_section: contacts
|
||||
section_title: Контакты
|
||||
language: ru
|
||||
---
|
||||
Some text here
|
||||
""".trimIndent()
|
||||
|
||||
val envelope = context.io.readEnvelope(text)
|
||||
assertEquals("Some text here", envelope.data!!.toByteArray().decodeToString().trim())
|
||||
assertEquals("magprog", envelope.meta["content_type"].string)
|
||||
}
|
||||
}
|
@ -1,6 +1,9 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.buffered
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlin.math.min
|
||||
|
||||
/**
|
||||
@ -12,11 +15,20 @@ public interface Binary {
|
||||
|
||||
public val size: Int
|
||||
|
||||
|
||||
/**
|
||||
* Read maximum of [atMost] bytes as input from the binary, starting at [offset]. The generated input is always closed
|
||||
* when leaving scope, so it could not be leaked outside of scope of [block].
|
||||
*/
|
||||
public fun <R> read(offset: Int = 0, atMost: Int = size - offset, block: Input.() -> R): R
|
||||
public fun <R> read(offset: Int = 0, atMost: Int = size - offset, block: Source.() -> R): R
|
||||
|
||||
public suspend fun <R> readSuspend(offset: Int = 0, atMost: Int = size - offset, block: suspend Source.() -> R): R
|
||||
|
||||
/**
|
||||
* Read a binary with given [offset] relative to this binary and given [binarySize].
|
||||
* In general, the resulting binary is of the same type as this one, but it is not guaranteed.
|
||||
*/
|
||||
public fun view(offset: Int, binarySize: Int = size - offset): Binary
|
||||
|
||||
public companion object {
|
||||
public val EMPTY: Binary = ByteArrayBinary(ByteArray(0))
|
||||
@ -29,46 +41,67 @@ internal class ByteArrayBinary(
|
||||
override val size: Int = array.size - start,
|
||||
) : Binary {
|
||||
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Source.() -> R): R {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset < array.size) { "Offset $offset is larger than array size" }
|
||||
val input = ByteReadPacket(
|
||||
|
||||
return ByteArraySource(
|
||||
array,
|
||||
offset + start,
|
||||
min(atMost, size - offset)
|
||||
)
|
||||
return input.use(block)
|
||||
).buffered().use(block)
|
||||
}
|
||||
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset < array.size) { "Offset $offset is larger than array size" }
|
||||
|
||||
val input = ByteArraySource(
|
||||
array,
|
||||
offset + start,
|
||||
min(atMost, size - offset)
|
||||
).buffered()
|
||||
|
||||
return try {
|
||||
block(input)
|
||||
} finally {
|
||||
input.close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun view(offset: Int, binarySize: Int): ByteArrayBinary =
|
||||
ByteArrayBinary(array, start + offset, binarySize)
|
||||
}
|
||||
|
||||
public fun ByteArray.asBinary(): Binary = ByteArrayBinary(this)
|
||||
|
||||
/**
|
||||
* Produce a [buildByteArray] representing an exact copy of this [Binary]
|
||||
* Produce a [ByteArray] representing an exact copy of this [Binary]
|
||||
*/
|
||||
public fun Binary.toByteArray(): ByteArray = if (this is ByteArrayBinary) {
|
||||
array.copyOf() // TODO do we need to ensure data safety here?
|
||||
array.copyOfRange(start, start + size) // TODO do we need to ensure data safety here?
|
||||
} else {
|
||||
read {
|
||||
readBytes()
|
||||
readByteArray()
|
||||
}
|
||||
}
|
||||
|
||||
public fun Input.readBinary(size: Int): Binary {
|
||||
val array = readBytes(size)
|
||||
//TODO optimize for file-based Inputs
|
||||
public fun Source.readBinary(size: Int? = null): Binary {
|
||||
val array = if (size == null) readByteArray() else readByteArray(size)
|
||||
return ByteArrayBinary(array)
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct write of binary to the output. Returns the number of bytes written
|
||||
*/
|
||||
public fun Output.writeBinary(binary: Binary): Int {
|
||||
public fun Sink.writeBinary(binary: Binary): Int {
|
||||
return if (binary is ByteArrayBinary) {
|
||||
writeFully(binary.array, binary.start, binary.start + binary.size)
|
||||
write(binary.array, binary.start, binary.start + binary.size)
|
||||
binary.size
|
||||
} else {
|
||||
binary.read {
|
||||
copyTo(this@writeBinary).toInt()
|
||||
transferTo(this@writeBinary).toInt()
|
||||
}
|
||||
}
|
||||
}
|
@ -1,133 +0,0 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.values.*
|
||||
|
||||
/**
|
||||
* A DataForge-specific simplified binary format for meta
|
||||
* TODO add description
|
||||
*/
|
||||
public object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
|
||||
override val shortName: String = "bin"
|
||||
override val key: Short = 0x4249//BI
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): MetaFormat = this
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
|
||||
return (input.readMetaItem() as MetaItemNode).node
|
||||
}
|
||||
|
||||
private fun Output.writeChar(char: Char) = writeByte(char.toByte())
|
||||
|
||||
private fun Output.writeString(str: String) {
|
||||
writeInt(str.length)
|
||||
writeFully(str.encodeToByteArray())
|
||||
}
|
||||
|
||||
public fun Output.writeValue(value: Value): Unit = when (value.type) {
|
||||
ValueType.NUMBER -> when (value.value) {
|
||||
is Short -> {
|
||||
writeChar('s')
|
||||
writeShort(value.short)
|
||||
}
|
||||
is Int -> {
|
||||
writeChar('i')
|
||||
writeInt(value.int)
|
||||
}
|
||||
is Long -> {
|
||||
writeChar('l')
|
||||
writeLong(value.long)
|
||||
}
|
||||
is Float -> {
|
||||
writeChar('f')
|
||||
writeFloat(value.float)
|
||||
}
|
||||
else -> {
|
||||
writeChar('d')
|
||||
writeDouble(value.double)
|
||||
}
|
||||
}
|
||||
ValueType.STRING -> {
|
||||
writeChar('S')
|
||||
writeString(value.string)
|
||||
}
|
||||
ValueType.BOOLEAN -> {
|
||||
if (value.boolean) {
|
||||
writeChar('+')
|
||||
} else {
|
||||
writeChar('-')
|
||||
}
|
||||
}
|
||||
ValueType.NULL -> {
|
||||
writeChar('N')
|
||||
}
|
||||
ValueType.LIST -> {
|
||||
writeChar('L')
|
||||
writeInt(value.list.size)
|
||||
value.list.forEach {
|
||||
writeValue(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun writeMeta(
|
||||
output: Output,
|
||||
meta: Meta,
|
||||
descriptor: space.kscience.dataforge.meta.descriptors.NodeDescriptor?,
|
||||
) {
|
||||
output.writeChar('M')
|
||||
output.writeInt(meta.items.size)
|
||||
meta.items.forEach { (key, item) ->
|
||||
output.writeString(key.toString())
|
||||
when (item) {
|
||||
is MetaItemValue -> {
|
||||
output.writeValue(item.value)
|
||||
}
|
||||
is MetaItemNode -> {
|
||||
writeObject(output, item.node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun Input.readString(): String {
|
||||
val length = readInt()
|
||||
val array = readBytes(length)
|
||||
return array.decodeToString()
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
public fun Input.readMetaItem(): TypedMetaItem<MetaBuilder> {
|
||||
return when (val keyChar = readByte().toChar()) {
|
||||
'S' -> MetaItemValue(StringValue(readString()))
|
||||
'N' -> MetaItemValue(Null)
|
||||
'+' -> MetaItemValue(True)
|
||||
'-' -> MetaItemValue(True)
|
||||
's' -> MetaItemValue(NumberValue(readShort()))
|
||||
'i' -> MetaItemValue(NumberValue(readInt()))
|
||||
'l' -> MetaItemValue(NumberValue(readInt()))
|
||||
'f' -> MetaItemValue(NumberValue(readFloat()))
|
||||
'd' -> MetaItemValue(NumberValue(readDouble()))
|
||||
'L' -> {
|
||||
val length = readInt()
|
||||
val list = (1..length).map { (readMetaItem() as MetaItemValue).value }
|
||||
MetaItemValue(Value.of(list))
|
||||
}
|
||||
'M' -> {
|
||||
val length = readInt()
|
||||
val meta = Meta {
|
||||
(1..length).forEach { _ ->
|
||||
val name = readString()
|
||||
val item = readMetaItem()
|
||||
set(name, item)
|
||||
}
|
||||
}
|
||||
MetaItemNode(meta)
|
||||
}
|
||||
else -> error("Unknown serialization key character: $keyChar")
|
||||
}
|
||||
}
|
||||
}
|
@ -34,7 +34,9 @@ public interface Envelope {
|
||||
}
|
||||
}
|
||||
|
||||
public class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
|
||||
internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
|
||||
|
||||
public fun Envelope(meta: Meta, data: Binary?): Envelope = SimpleEnvelope(meta, data)
|
||||
|
||||
/**
|
||||
* The purpose of the envelope
|
||||
|
@ -1,10 +1,10 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Sink
|
||||
import space.kscience.dataforge.meta.*
|
||||
|
||||
public class EnvelopeBuilder : Envelope {
|
||||
private val metaBuilder = MetaBuilder()
|
||||
private val metaBuilder = MutableMeta()
|
||||
|
||||
override var data: Binary? = null
|
||||
override var meta: Meta
|
||||
@ -13,7 +13,7 @@ public class EnvelopeBuilder : Envelope {
|
||||
metaBuilder.update(value)
|
||||
}
|
||||
|
||||
public fun meta(block: MetaBuilder.() -> Unit) {
|
||||
public fun meta(block: MutableMeta.() -> Unit) {
|
||||
metaBuilder.apply(block)
|
||||
}
|
||||
|
||||
@ -33,8 +33,8 @@ public class EnvelopeBuilder : Envelope {
|
||||
/**
|
||||
* Construct a data binary from given builder
|
||||
*/
|
||||
public fun data(block: Output.() -> Unit) {
|
||||
data = buildByteArray { block() }.asBinary()
|
||||
public inline fun data(block: Sink.() -> Unit) {
|
||||
data = ByteArray { block() }.asBinary()
|
||||
}
|
||||
|
||||
public fun seal(): Envelope = SimpleEnvelope(metaBuilder.seal(), data)
|
||||
|
@ -1,48 +1,27 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Source
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
/**
|
||||
* A partially read envelope with meta, but without data
|
||||
*/
|
||||
public data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
|
||||
|
||||
public interface EnvelopeFormat : IOFormat<Envelope> {
|
||||
|
||||
override val type: KType get() = typeOf<Envelope>()
|
||||
|
||||
public val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
|
||||
|
||||
public fun readPartial(input: Input): PartialEnvelope
|
||||
|
||||
public fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory = defaultMetaFormat,
|
||||
formatMeta: Meta = Meta.EMPTY,
|
||||
)
|
||||
|
||||
override fun readObject(input: Input): Envelope
|
||||
|
||||
override fun writeObject(output: Output, obj: Envelope): Unit = writeEnvelope(output, obj)
|
||||
}
|
||||
|
||||
public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input)
|
||||
public fun EnvelopeFormat.read(input: Source): Envelope = readFrom(input)
|
||||
|
||||
@Type(ENVELOPE_FORMAT_TYPE)
|
||||
@DfId(ENVELOPE_FORMAT_TYPE)
|
||||
public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
|
||||
override val name: Name get() = "envelope".asName()
|
||||
override val type: KType get() = typeOf<Envelope>()
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat
|
||||
|
||||
/**
|
||||
* Try to infer specific format from input and return null if the attempt is failed.
|
||||
@ -51,6 +30,7 @@ public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeForm
|
||||
public fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat?
|
||||
|
||||
public companion object {
|
||||
public val ENVELOPE_FACTORY_NAME: Name = "envelope".asName()
|
||||
public const val ENVELOPE_FORMAT_TYPE: String = "io.format.envelope"
|
||||
}
|
||||
}
|
@ -1,30 +1,29 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import kotlinx.io.write
|
||||
import space.kscience.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.PARTS_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.PART_FORMAT_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.SEPARATOR_KEY
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.plus
|
||||
import space.kscience.dataforge.names.toName
|
||||
|
||||
private class PartDescriptor : Scheme() {
|
||||
var offset by int(0)
|
||||
var size by int(0)
|
||||
var partMeta by node("meta".toName())
|
||||
var partMeta by node("meta".asName())
|
||||
|
||||
companion object : SchemeSpec<PartDescriptor>(::PartDescriptor) {
|
||||
val MULTIPART_KEY = ENVELOPE_NODE_KEY + "multipart"
|
||||
val PARTS_KEY = MULTIPART_KEY + "parts"
|
||||
val SEPARATOR_KEY = MULTIPART_KEY + "separator"
|
||||
|
||||
const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n"
|
||||
|
||||
val PART_FORMAT_KEY = "format".asName()
|
||||
val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n".toAsciiByteString()
|
||||
|
||||
const val MULTIPART_DATA_TYPE = "envelope.multipart"
|
||||
}
|
||||
@ -36,12 +35,12 @@ public typealias EnvelopeParts = List<EnvelopePart>
|
||||
|
||||
public fun EnvelopeBuilder.multipart(
|
||||
parts: EnvelopeParts,
|
||||
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
|
||||
separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
) {
|
||||
dataType = MULTIPART_DATA_TYPE
|
||||
|
||||
var offsetCounter = 0
|
||||
val separatorSize = separator.length
|
||||
val separatorSize = separator.size
|
||||
val partDescriptors = parts.map { (binary, description) ->
|
||||
offsetCounter += separatorSize
|
||||
PartDescriptor {
|
||||
@ -55,46 +54,45 @@ public fun EnvelopeBuilder.multipart(
|
||||
|
||||
meta {
|
||||
if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) {
|
||||
SEPARATOR_KEY put separator
|
||||
SEPARATOR_KEY put separator.decodeToString()
|
||||
}
|
||||
setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() })
|
||||
}
|
||||
|
||||
data {
|
||||
parts.forEach {
|
||||
writeRawString(separator)
|
||||
write(separator)
|
||||
writeBinary(it.binary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a list of envelopes as parts of given envelope
|
||||
*/
|
||||
public fun EnvelopeBuilder.envelopes(
|
||||
envelopes: List<Envelope>,
|
||||
format: EnvelopeFormat = TaggedEnvelopeFormat,
|
||||
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
|
||||
separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
) {
|
||||
val parts = envelopes.map {
|
||||
val binary = format.toBinary(it)
|
||||
val binary = Binary(it, TaggedEnvelopeFormat)
|
||||
EnvelopePart(binary, null)
|
||||
}
|
||||
meta{
|
||||
set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta())
|
||||
}
|
||||
multipart(parts, separator)
|
||||
}
|
||||
|
||||
public fun Envelope.parts(): EnvelopeParts {
|
||||
if (data == null) return emptyList()
|
||||
//TODO add zip folder reader
|
||||
val parts = meta.getIndexed(PARTS_KEY).values.mapNotNull { it.node }.map {
|
||||
val parts = meta.getIndexed(PARTS_KEY).values.map {
|
||||
PartDescriptor.read(it)
|
||||
}
|
||||
return if (parts.isEmpty()) {
|
||||
listOf(EnvelopePart(data!!, meta[MULTIPART_KEY].node))
|
||||
listOf(EnvelopePart(data!!, meta[MULTIPART_KEY]))
|
||||
} else {
|
||||
parts.map {
|
||||
val binary = data!!.view(it.offset, it.size)
|
||||
val meta = Laminate(it.partMeta, meta[MULTIPART_KEY].node)
|
||||
val meta = Laminate(it.partMeta, meta[MULTIPART_KEY])
|
||||
EnvelopePart(binary, meta)
|
||||
}
|
||||
}
|
||||
@ -107,14 +105,4 @@ public val EnvelopePart.name: String? get() = description?.get("name").string
|
||||
/**
|
||||
* Represent envelope part by an envelope
|
||||
*/
|
||||
public fun EnvelopePart.envelope(plugin: IOPlugin): Envelope {
|
||||
val formatItem = description?.get(PART_FORMAT_KEY)
|
||||
return if (formatItem != null) {
|
||||
val format: EnvelopeFormat = plugin.resolveEnvelopeFormat(formatItem)
|
||||
?: error("Envelope format for $formatItem is not resolved")
|
||||
binary.readWith(format)
|
||||
} else {
|
||||
error("Envelope description not found")
|
||||
//SimpleEnvelope(description ?: Meta.EMPTY, binary)
|
||||
}
|
||||
}
|
||||
public fun EnvelopePart.envelope(): Envelope = binary.readWith(TaggedEnvelopeFormat)
|
@ -1,135 +1,118 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readByteArray
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Factory
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaItemValue
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.values.Value
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
/**
|
||||
* And interface for reading and writing objects into with IO streams
|
||||
* Reader of a custom object from input
|
||||
*/
|
||||
public interface IOFormat<T : Any> : MetaRepr {
|
||||
public interface IOReader<out T> {
|
||||
/**
|
||||
* The type of object being read
|
||||
*/
|
||||
public val type: KType
|
||||
|
||||
public fun writeObject(output: Output, obj: T)
|
||||
public fun readObject(input: Input): T
|
||||
public fun readFrom(source: Source): T
|
||||
|
||||
public fun readFrom(binary: Binary): T = binary.read { readFrom(this) }
|
||||
|
||||
public companion object {
|
||||
public val NAME_KEY: Name = "name".asName()
|
||||
public val META_KEY: Name = "meta".asName()
|
||||
/**
|
||||
* no-op reader for binaries.
|
||||
*/
|
||||
public val binary: IOReader<Binary> = object : IOReader<Binary> {
|
||||
override val type: KType = typeOf<Binary>()
|
||||
|
||||
override fun readFrom(source: Source): Binary = source.readByteArray().asBinary()
|
||||
|
||||
override fun readFrom(binary: Binary): Binary = binary
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.readObject(this@readWith)
|
||||
public inline fun <reified T> IOReader(crossinline read: Source.() -> T): IOReader<T> = object : IOReader<T> {
|
||||
override val type: KType = typeOf<T>()
|
||||
|
||||
public fun <T: Any> IOFormat<T>.readObject(binary: Binary): T = binary.read {
|
||||
readObject(this)
|
||||
override fun readFrom(source: Source): T = source.read()
|
||||
}
|
||||
|
||||
public fun interface IOWriter<in T> {
|
||||
public fun writeTo(sink: Sink, obj: T)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read given binary as object using given format
|
||||
* And interface for reading and writing objects into with IO streams
|
||||
*/
|
||||
public fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
|
||||
public interface IOFormat<T> : IOReader<T>, IOWriter<T>
|
||||
|
||||
public fun <T : Any> Source.readWith(format: IOReader<T>): T = format.readFrom(this)
|
||||
|
||||
/**
|
||||
* Read given binary as an object using given format
|
||||
*/
|
||||
public fun <T : Any> Binary.readWith(format: IOReader<T>): T = read {
|
||||
readWith(format)
|
||||
}
|
||||
|
||||
public fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T): Unit =
|
||||
format.run { writeObject(this@writeWith, obj) }
|
||||
/**
|
||||
* Write an object to the [Sink] with given [format]
|
||||
*/
|
||||
public fun <T : Any> Sink.writeWith(format: IOWriter<T>, obj: T): Unit =
|
||||
format.writeTo(this, obj)
|
||||
|
||||
public inline fun <reified T : Any> IOFormat.Companion.listOf(
|
||||
format: IOFormat<T>,
|
||||
): IOFormat<List<T>> = object : IOFormat<List<T>> {
|
||||
override val type: KType = typeOf<List<T>>()
|
||||
|
||||
override fun writeObject(output: Output, obj: List<T>) {
|
||||
output.writeInt(obj.size)
|
||||
format.run {
|
||||
obj.forEach {
|
||||
writeObject(output, it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): List<T> {
|
||||
val size = input.readInt()
|
||||
return format.run {
|
||||
List(size) { readObject(input) }
|
||||
}
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put "list"
|
||||
"contentFormat" put format.toMeta()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//public fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
|
||||
// val buffer = borrow()
|
||||
// return try {
|
||||
// buffer.apply(block)
|
||||
// } catch (ex: Exception) {
|
||||
// //recycle(buffer)
|
||||
// throw ex
|
||||
// }
|
||||
//}
|
||||
|
||||
@Type(IO_FORMAT_TYPE)
|
||||
public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
|
||||
@DfId(IO_FORMAT_TYPE)
|
||||
public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
|
||||
/**
|
||||
* Explicit type for dynamic type checks
|
||||
*/
|
||||
public val type: KType
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
}
|
||||
|
||||
public companion object {
|
||||
public const val IO_FORMAT_TYPE: String = "io.format"
|
||||
public val NAME_KEY: Name = "name".asName()
|
||||
public val META_KEY: Name = "meta".asName()
|
||||
}
|
||||
}
|
||||
|
||||
public fun <T : Any> IOFormat<T>.toBinary(obj: T): Binary = Binary { writeObject(this, obj) }
|
||||
public fun <T : Any> Binary(obj: T, format: IOWriter<T>): Binary = Binary { format.writeTo(this, obj) }
|
||||
|
||||
public object FloatIOFormat : IOFormat<Float>, IOFormatFactory<Float> {
|
||||
override fun build(context: Context, meta: Meta): IOFormat<Float> = this
|
||||
|
||||
override val name: Name = "float32".asName()
|
||||
|
||||
override val type: KType get() = typeOf<Float>()
|
||||
|
||||
override fun writeTo(sink: Sink, obj: Float) {
|
||||
sink.writeFloat(obj)
|
||||
}
|
||||
|
||||
override fun readFrom(source: Source): Float = source.readFloat()
|
||||
}
|
||||
|
||||
|
||||
public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
|
||||
override fun invoke(meta: Meta, context: Context): IOFormat<Double> = this
|
||||
override fun build(context: Context, meta: Meta): IOFormat<Double> = this
|
||||
|
||||
override val name: Name = "double".asName()
|
||||
override val name: Name = "float64".asName()
|
||||
|
||||
override val type: KType get() = typeOf<Double>()
|
||||
|
||||
override fun writeObject(output: Output, obj: kotlin.Double) {
|
||||
output.writeDouble(obj)
|
||||
override fun writeTo(sink: Sink, obj: Double) {
|
||||
sink.writeLong(obj.toBits())
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Double = input.readDouble()
|
||||
}
|
||||
|
||||
public object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
|
||||
override fun invoke(meta: Meta, context: Context): IOFormat<Value> = this
|
||||
|
||||
override val name: Name = "value".asName()
|
||||
|
||||
override val type: KType get() = typeOf<Value>()
|
||||
|
||||
override fun writeObject(output: Output, obj: Value) {
|
||||
BinaryMetaFormat.run { output.writeValue(obj) }
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Value {
|
||||
return (BinaryMetaFormat.run { input.readMetaItem() } as? MetaItemValue)?.value
|
||||
?: error("The item is not a value")
|
||||
}
|
||||
override fun readFrom(source: Source): Double = source.readDouble()
|
||||
}
|
@ -2,14 +2,15 @@ package space.kscience.dataforge.io
|
||||
|
||||
import space.kscience.dataforge.context.*
|
||||
import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
||||
import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.reflect.KClass
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
@ -18,15 +19,14 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
context.gather<IOFormatFactory<*>>(IO_FORMAT_TYPE).values
|
||||
}
|
||||
|
||||
public fun <T : Any> resolveIOFormat(item: MetaItem, type: KClass<out T>): IOFormat<T>? {
|
||||
val key = item.string ?: item.node[NAME_KEY]?.string ?: error("Format name not defined")
|
||||
val name = key.toName()
|
||||
return ioFormatFactories.find { it.name == name }?.let {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
|
||||
else it.invoke(item.node[META_KEY].node ?: Meta.EMPTY, context) as IOFormat<T>
|
||||
}
|
||||
}
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
@DFInternal
|
||||
public fun <T : Any> resolveIOFormat(type: KType, meta: Meta): IOFormat<T>? =
|
||||
ioFormatFactories.singleOrNull { it.type == type }?.build(context, meta) as? IOFormat<T>
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline fun <reified T : Any> resolveIOFormat(meta: Meta = Meta.EMPTY): IOFormat<T>? =
|
||||
resolveIOFormat(typeOf<T>(), meta)
|
||||
|
||||
|
||||
public val metaFormatFactories: Collection<MetaFormatFactory> by lazy {
|
||||
@ -34,41 +34,53 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
}
|
||||
|
||||
public fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? =
|
||||
metaFormatFactories.find { it.key == key }?.invoke(meta)
|
||||
metaFormatFactories.find { it.key == key }?.build(context, meta)
|
||||
|
||||
public fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? =
|
||||
metaFormatFactories.find { it.shortName == name }?.invoke(meta)
|
||||
metaFormatFactories.find { it.shortName == name }?.build(context, meta)
|
||||
|
||||
public val envelopeFormatFactories: Collection<EnvelopeFormatFactory> by lazy {
|
||||
context.gather<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
|
||||
}
|
||||
|
||||
private fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? =
|
||||
envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
|
||||
envelopeFormatFactories.find { it.name == name }?.build(context, meta)
|
||||
|
||||
public fun resolveEnvelopeFormat(item: MetaItem): EnvelopeFormat? {
|
||||
val name = item.string ?: item.node[NAME_KEY]?.string ?: error("Envelope format name not defined")
|
||||
val meta = item.node[META_KEY].node ?: Meta.EMPTY
|
||||
return resolveEnvelopeFormat(name.toName(), meta)
|
||||
public fun resolveEnvelopeFormat(item: Meta): EnvelopeFormat? {
|
||||
val name = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Envelope format name not defined")
|
||||
val meta = item[IOFormatFactory.META_KEY] ?: Meta.EMPTY
|
||||
return resolveEnvelopeFormat(Name.parse(name), meta)
|
||||
}
|
||||
|
||||
override fun content(target: String): Map<Name, Any> {
|
||||
return when (target) {
|
||||
META_FORMAT_TYPE -> defaultMetaFormats.toMap()
|
||||
ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap()
|
||||
else -> super.content(target)
|
||||
}
|
||||
override fun content(target: String): Map<Name, Any> = when (target) {
|
||||
META_FORMAT_TYPE -> defaultMetaFormats.associateByName()
|
||||
ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.associateByName()
|
||||
IO_FORMAT_TYPE -> content(META_FORMAT_TYPE) + content(ENVELOPE_FORMAT_TYPE)
|
||||
else -> super.content(target)
|
||||
}
|
||||
|
||||
public companion object : PluginFactory<IOPlugin> {
|
||||
public val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
|
||||
public val defaultEnvelopeFormats: List<EnvelopeFormatFactory> = listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
|
||||
public val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat)
|
||||
public val defaultEnvelopeFormats: List<EnvelopeFormatFactory> = listOf(
|
||||
TaggedEnvelopeFormat,
|
||||
TaglessEnvelopeFormat
|
||||
)
|
||||
|
||||
override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
|
||||
|
||||
override val type: KClass<out IOPlugin> = IOPlugin::class
|
||||
override fun invoke(meta: Meta, context: Context): IOPlugin = IOPlugin(meta)
|
||||
override fun build(context: Context, meta: Meta): IOPlugin = IOPlugin(meta)
|
||||
|
||||
public val WORK_DIRECTORY_KEY: Name = Name.of("io", "workDirectory")
|
||||
}
|
||||
}
|
||||
|
||||
public val Context.io: IOPlugin get() = plugins.fetch(IOPlugin)
|
||||
internal val ioContext = Context("IO") {
|
||||
plugin(IOPlugin)
|
||||
}
|
||||
|
||||
public val Context.io: IOPlugin
|
||||
get() = if (this == Global) {
|
||||
ioContext.request(IOPlugin)
|
||||
} else {
|
||||
request(IOPlugin)
|
||||
}
|
@ -1,59 +1,48 @@
|
||||
@file:Suppress("UNUSED_PARAMETER")
|
||||
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readString
|
||||
import kotlinx.io.writeString
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.meta.node
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.toJson
|
||||
import space.kscience.dataforge.meta.toMetaItem
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
import space.kscience.dataforge.meta.toMeta
|
||||
|
||||
/**
|
||||
* A Json format for Meta representation
|
||||
*/
|
||||
public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
|
||||
|
||||
override val type: KType get() = typeOf<Meta>()
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) {
|
||||
val jsonObject = meta.toJson(descriptor)
|
||||
output.writeUtf8String(json.encodeToString(JsonObject.serializer(), jsonObject))
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
val jsonElement = meta.toJson(descriptor)
|
||||
sink.writeString(json.encodeToString(JsonElement.serializer(), jsonElement))
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
}
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
|
||||
val str = input.readUtf8String()//readByteArray().decodeToString()
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta {
|
||||
val str = source.readString()
|
||||
val jsonElement = json.parseToJsonElement(str)
|
||||
val item = jsonElement.toMetaItem(descriptor)
|
||||
return item.node ?: Meta.EMPTY
|
||||
return jsonElement.toMeta(descriptor)
|
||||
}
|
||||
|
||||
public companion object : MetaFormatFactory {
|
||||
public val DEFAULT_JSON: Json = Json { prettyPrint = true }
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): MetaFormat = default
|
||||
override fun build(context: Context, meta: Meta): MetaFormat = default
|
||||
|
||||
override val shortName: String = "json"
|
||||
override val key: Short = 0x4a53//"JS"
|
||||
|
||||
private val default = JsonMetaFormat()
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit =
|
||||
default.run { writeMeta(output, meta, descriptor) }
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.run { writeMeta(sink, meta, descriptor) }
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta =
|
||||
default.run { readMeta(input, descriptor) }
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
default.run { readMeta(source, descriptor) }
|
||||
}
|
||||
}
|
||||
|
@ -1,14 +1,15 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import io.ktor.utils.io.core.use
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.buffered
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.plus
|
||||
@ -19,24 +20,25 @@ import kotlin.reflect.typeOf
|
||||
* A format for meta serialization
|
||||
*/
|
||||
public interface MetaFormat : IOFormat<Meta> {
|
||||
|
||||
override val type: KType get() = typeOf<Meta>()
|
||||
|
||||
override fun writeObject(output: Output, obj: Meta) {
|
||||
writeMeta(output, obj, null)
|
||||
override fun writeTo(sink: Sink, obj: Meta) {
|
||||
writeMeta(sink, obj, null)
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Meta = readMeta(input)
|
||||
override fun readFrom(source: Source): Meta = readMeta(source)
|
||||
|
||||
public fun writeMeta(
|
||||
output: Output,
|
||||
sink: Sink,
|
||||
meta: Meta,
|
||||
descriptor: NodeDescriptor? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
)
|
||||
|
||||
public fun readMeta(input: Input, descriptor: NodeDescriptor? = null): Meta
|
||||
public fun readMeta(source: Source, descriptor: MetaDescriptor? = null): Meta
|
||||
}
|
||||
|
||||
@Type(META_FORMAT_TYPE)
|
||||
@DfId(META_FORMAT_TYPE)
|
||||
public interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
|
||||
public val shortName: String
|
||||
|
||||
@ -46,25 +48,23 @@ public interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
|
||||
|
||||
public val key: Short get() = name.hashCode().toShort()
|
||||
|
||||
override operator fun invoke(meta: Meta, context: Context): MetaFormat
|
||||
override fun build(context: Context, meta: Meta): MetaFormat
|
||||
|
||||
public companion object {
|
||||
public const val META_FORMAT_TYPE: String = "io.format.meta"
|
||||
}
|
||||
}
|
||||
|
||||
public fun Meta.toString(format: MetaFormat): String = buildByteArray {
|
||||
public fun Meta.toString(format: MetaFormat): String = ByteArray {
|
||||
format.run {
|
||||
writeObject(this@buildByteArray, this@toString)
|
||||
writeTo(this@ByteArray, this@toString)
|
||||
}
|
||||
}.decodeToString()
|
||||
|
||||
public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
|
||||
public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory.build(Global, Meta.EMPTY))
|
||||
|
||||
public fun MetaFormat.parse(str: String): Meta {
|
||||
return ByteReadPacket(str.encodeToByteArray()).use { readObject(it) }
|
||||
}
|
||||
public fun MetaFormat.parse(str: String): Meta = readFrom(StringSource(str).buffered())
|
||||
|
||||
public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = invoke(formatMeta).parse(str)
|
||||
public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = build(Global, formatMeta).parse(str)
|
||||
|
||||
|
||||
|
@ -1,16 +1,16 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.enum
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
import space.kscience.dataforge.names.toName
|
||||
|
||||
|
||||
/**
|
||||
* A streaming-friendly envelope format with a short binary tag.
|
||||
@ -19,79 +19,79 @@ import space.kscience.dataforge.names.toName
|
||||
public class TaggedEnvelopeFormat(
|
||||
public val io: IOPlugin,
|
||||
public val version: VERSION = VERSION.DF02,
|
||||
public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat,
|
||||
) : EnvelopeFormat {
|
||||
|
||||
// private val metaFormat = io.metaFormat(metaFormatKey)
|
||||
// ?: error("Meta format with key $metaFormatKey could not be resolved in $io")
|
||||
|
||||
|
||||
private fun Tag.toBinary() = Binary(24) {
|
||||
writeRawString(START_SEQUENCE)
|
||||
writeRawString(version.name)
|
||||
private fun Tag.toBinary() = Binary {
|
||||
write(START_SEQUENCE)
|
||||
writeString(version.name)
|
||||
writeShort(metaFormatKey)
|
||||
writeUInt(metaSize)
|
||||
when (version) {
|
||||
VERSION.DF02 -> {
|
||||
writeUInt(dataSize.toUInt())
|
||||
}
|
||||
|
||||
VERSION.DF03 -> {
|
||||
writeULong(dataSize)
|
||||
}
|
||||
}
|
||||
writeRawString(END_SEQUENCE)
|
||||
write(END_SEQUENCE)
|
||||
}
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory.invoke(formatMeta, this@TaggedEnvelopeFormat.io.context)
|
||||
val metaBytes = metaFormat.toBinary(envelope.meta)
|
||||
val actualSize: ULong = (envelope.data?.size ?: 0).toULong()
|
||||
val metaFormat = metaFormatFactory.build(io.context, Meta.EMPTY)
|
||||
val metaBytes = Binary(obj.meta, metaFormat)
|
||||
val actualSize: ULong = (obj.data?.size ?: 0).toULong()
|
||||
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
|
||||
output.writeBinary(tag.toBinary())
|
||||
output.writeBinary(metaBytes)
|
||||
output.writeRawString("\r\n")
|
||||
envelope.data?.let {
|
||||
output.writeBinary(it)
|
||||
sink.writeBinary(tag.toBinary())
|
||||
sink.writeBinary(metaBytes)
|
||||
sink.writeString("\r\n")
|
||||
obj.data?.let {
|
||||
sink.writeBinary(it)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an envelope from input into memory
|
||||
*
|
||||
* @param input an input to read from
|
||||
* @param source an input to read from
|
||||
* @param formats a collection of meta formats to resolve
|
||||
*/
|
||||
override fun readObject(input: Input): Envelope {
|
||||
val tag = input.readTag(this.version)
|
||||
override fun readFrom(source: Source): Envelope {
|
||||
val tag = source.readTag(this.version)
|
||||
|
||||
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
|
||||
?: error("Meta format with key ${tag.metaFormatKey} not found")
|
||||
|
||||
val metaBinary = input.readBinary(tag.metaSize.toInt())
|
||||
val metaBinary = source.readBinary(tag.metaSize.toInt())
|
||||
|
||||
val meta: Meta = metaFormat.readObject(metaBinary)
|
||||
val meta: Meta = metaFormat.readFrom(metaBinary)
|
||||
|
||||
val data = input.readBinary(tag.dataSize.toInt())
|
||||
val data = source.readBinary(tag.dataSize.toInt())
|
||||
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope {
|
||||
val tag = input.readTag(this.version)
|
||||
override fun readFrom(binary: Binary): Envelope = binary.read {
|
||||
val tag = readTag(version)
|
||||
|
||||
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
|
||||
?: error("Meta format with key ${tag.metaFormatKey} not found")
|
||||
|
||||
val metaBinary = input.readBinary(tag.metaSize.toInt())
|
||||
val metaBinary = readBinary(tag.metaSize.toInt())
|
||||
|
||||
val meta: Meta = metaFormat.readObject(metaBinary)
|
||||
val meta: Meta = metaFormat.readFrom(metaBinary)
|
||||
|
||||
|
||||
return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize)
|
||||
SimpleEnvelope(meta, binary.view((version.tagSize + tag.metaSize).toInt(), tag.dataSize.toInt()))
|
||||
}
|
||||
|
||||
private data class Tag(
|
||||
@ -105,23 +105,16 @@ public class TaggedEnvelopeFormat(
|
||||
DF03(24u)
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
META_KEY put {
|
||||
"version" put version
|
||||
}
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
private const val START_SEQUENCE = "#~"
|
||||
private const val END_SEQUENCE = "~#\r\n"
|
||||
private val START_SEQUENCE = "#~".toAsciiByteString()
|
||||
private val END_SEQUENCE = "~#\r\n".toAsciiByteString()
|
||||
|
||||
override val name: Name = super.name + "tagged"
|
||||
override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "tagged"
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat {
|
||||
val io = context.io
|
||||
|
||||
val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name
|
||||
val metaFormatName = meta["name"].string?.let { Name.parse(it) } ?: JsonMetaFormat.name
|
||||
//Check if appropriate factory exists
|
||||
io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved")
|
||||
|
||||
@ -130,57 +123,48 @@ public class TaggedEnvelopeFormat(
|
||||
return TaggedEnvelopeFormat(io, version)
|
||||
}
|
||||
|
||||
private fun Input.readTag(version: VERSION): Tag {
|
||||
val start = readRawString(2)
|
||||
private fun Source.readTag(version: VERSION): Tag {
|
||||
val start = readByteString(2)
|
||||
if (start != START_SEQUENCE) error("The input is not an envelope")
|
||||
val versionString = readRawString(4)
|
||||
if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
|
||||
val versionString = readByteString(4)
|
||||
if (version.name.toAsciiByteString() != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
|
||||
val metaFormatKey = readShort()
|
||||
val metaLength = readUInt()
|
||||
val dataLength: ULong = when (version) {
|
||||
VERSION.DF02 -> readUInt().toULong()
|
||||
VERSION.DF03 -> readULong()
|
||||
}
|
||||
val end = readRawString(4)
|
||||
val end = readByteString(4)
|
||||
if (end != END_SEQUENCE) error("The input is not an envelope")
|
||||
return Tag(metaFormatKey, metaLength, dataLength)
|
||||
}
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
|
||||
return try {
|
||||
binary.read{
|
||||
val header = readRawString(6)
|
||||
return@read when (header.substring(2..5)) {
|
||||
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
|
||||
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
|
||||
else -> null
|
||||
}
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = try {
|
||||
binary.read {
|
||||
val header = readByteString(6)
|
||||
when (header.substring(2, 6).decodeToString()) {
|
||||
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
|
||||
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
|
||||
else -> null
|
||||
}
|
||||
} catch (ex: Exception) {
|
||||
null
|
||||
}
|
||||
} catch (ex: Exception) {
|
||||
null
|
||||
}
|
||||
|
||||
private val default by lazy { invoke() }
|
||||
private val default by lazy { build(Global, Meta.EMPTY) }
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope =
|
||||
default.run { readPartial(input) }
|
||||
override fun readFrom(binary: Binary): Envelope =
|
||||
default.run { readFrom(binary) }
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
): Unit = default.run {
|
||||
writeEnvelope(
|
||||
output,
|
||||
envelope,
|
||||
metaFormatFactory,
|
||||
formatMeta
|
||||
)
|
||||
writeTo(sink, obj)
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope = default.readObject(input)
|
||||
override fun readFrom(source: Source): Envelope = default.readFrom(source)
|
||||
}
|
||||
|
||||
}
|
@ -1,175 +1,104 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.isEmpty
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import kotlin.collections.set
|
||||
import space.kscience.dataforge.names.plus
|
||||
|
||||
/**
|
||||
* A text envelope format with human-readable tag.
|
||||
* A text envelope format based on block separators.
|
||||
* TODO add description
|
||||
*/
|
||||
public class TaglessEnvelopeFormat(
|
||||
public val io: IOPlugin,
|
||||
public val meta: Meta = Meta.EMPTY,
|
||||
public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat,
|
||||
) : EnvelopeFormat {
|
||||
|
||||
private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
|
||||
private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START
|
||||
// private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
|
||||
// private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START
|
||||
|
||||
private fun Output.writeProperty(key: String, value: Any) {
|
||||
writeFully("#? $key: $value;\r\n".encodeToByteArray())
|
||||
}
|
||||
// private fun Output.writeProperty(key: String, value: Any) {
|
||||
// writeFully("#? $key: $value;\r\n".encodeToByteArray())
|
||||
// }
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory(formatMeta, this.io.context)
|
||||
val metaFormat = metaFormatFactory.build(this.io.context, meta)
|
||||
|
||||
//printing header
|
||||
output.writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n")
|
||||
|
||||
//printing all properties
|
||||
output.writeProperty(META_TYPE_PROPERTY,
|
||||
metaFormatFactory.shortName)
|
||||
//TODO add optional metaFormat properties
|
||||
val actualSize: Int = envelope.data?.size ?: 0
|
||||
|
||||
output.writeProperty(DATA_LENGTH_PROPERTY, actualSize)
|
||||
sink.write(TAGLESS_ENVELOPE_HEADER)
|
||||
sink.writeString("\r\n")
|
||||
|
||||
//Printing meta
|
||||
if (!envelope.meta.isEmpty()) {
|
||||
val metaBytes = metaFormat.toBinary(envelope.meta)
|
||||
output.writeProperty(META_LENGTH_PROPERTY,
|
||||
metaBytes.size + 2)
|
||||
output.writeUtf8String(this.metaStart + "\r\n")
|
||||
output.writeBinary(metaBytes)
|
||||
output.writeRawString("\r\n")
|
||||
if (!obj.meta.isEmpty()) {
|
||||
val metaBinary = Binary(obj.meta, metaFormat)
|
||||
sink.writeString(META_START + "-${metaFormatFactory.shortName}\r\n")
|
||||
sink.writeBinary(metaBinary)
|
||||
sink.writeString("\r\n")
|
||||
}
|
||||
|
||||
//Printing data
|
||||
envelope.data?.let { data ->
|
||||
output.writeUtf8String(this.dataStart + "\r\n")
|
||||
output.writeBinary(data)
|
||||
obj.data?.let { data ->
|
||||
//val actualSize: Int = envelope.data?.size ?: 0
|
||||
sink.writeString(DATA_START + "\r\n")
|
||||
sink.writeBinary(data)
|
||||
}
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope {
|
||||
var line: String
|
||||
do {
|
||||
line = input.readSafeUtf8Line() // ?: error("Input does not contain tagless envelope header")
|
||||
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
|
||||
val properties = HashMap<String, String>()
|
||||
|
||||
line = ""
|
||||
while (line.isBlank() || line.startsWith("#?")) {
|
||||
if (line.startsWith("#?")) {
|
||||
val match = propertyPattern.find(line)
|
||||
?: error("Line $line does not match property declaration pattern")
|
||||
val (key, value) = match.destructured
|
||||
properties[key] = value
|
||||
}
|
||||
//If can't read line, return envelope without data
|
||||
if (input.endOfInput) return SimpleEnvelope(Meta.EMPTY, null)
|
||||
line = input.readSafeUtf8Line()
|
||||
}
|
||||
override fun readFrom(source: Source): Envelope {
|
||||
//read preamble
|
||||
source.discardWithSeparator(
|
||||
TAGLESS_ENVELOPE_HEADER,
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
var meta: Meta = Meta.EMPTY
|
||||
|
||||
if (line.startsWith(metaStart)) {
|
||||
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
|
||||
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
|
||||
meta = if (metaSize != null) {
|
||||
metaFormat.readObject(input.readBinary(metaSize))
|
||||
} else {
|
||||
metaFormat.readObject(input)
|
||||
var data: Binary? = null
|
||||
|
||||
source.discardWithSeparator(
|
||||
SEPARATOR_PREFIX,
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
var header: String = ByteArray {
|
||||
source.readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
}.decodeToString()
|
||||
|
||||
while (!source.exhausted()) {
|
||||
val block = ByteArray {
|
||||
source.readWithSeparatorTo(this, SEPARATOR_PREFIX)
|
||||
}
|
||||
|
||||
val nextHeader = ByteArray {
|
||||
source.readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
}.decodeToString()
|
||||
|
||||
//terminate on end
|
||||
if (header.startsWith("END")) break
|
||||
|
||||
|
||||
if (header.startsWith("META")) {
|
||||
//TODO check format
|
||||
val metaFormat: MetaFormatFactory = JsonMetaFormat
|
||||
meta = metaFormat.readMeta(ByteArraySource(block).buffered())
|
||||
}
|
||||
|
||||
if (header.startsWith("DATA")) {
|
||||
data = block.asBinary()
|
||||
}
|
||||
header = nextHeader
|
||||
}
|
||||
|
||||
do {
|
||||
try {
|
||||
line = input.readSafeUtf8Line()
|
||||
} catch (ex: EOFException) {
|
||||
//returning an Envelope without data if end of input is reached
|
||||
return SimpleEnvelope(meta, null)
|
||||
}
|
||||
} while (!line.startsWith(dataStart))
|
||||
|
||||
val data: Binary = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
|
||||
input.readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt())
|
||||
// val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
|
||||
// readByteArray(bytes)
|
||||
// bytes.asBinary()
|
||||
} else {
|
||||
Binary {
|
||||
input.copyTo(this)
|
||||
}
|
||||
}
|
||||
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope {
|
||||
var offset = 0u
|
||||
var line: String
|
||||
do {
|
||||
line = input.readSafeUtf8Line()// ?: error("Input does not contain tagless envelope header")
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
|
||||
val properties = HashMap<String, String>()
|
||||
|
||||
line = ""
|
||||
while (line.isBlank() || line.startsWith("#?")) {
|
||||
if (line.startsWith("#?")) {
|
||||
val match = propertyPattern.find(line)
|
||||
?: error("Line $line does not match property declaration pattern")
|
||||
val (key, value) = match.destructured
|
||||
properties[key] = value
|
||||
}
|
||||
try {
|
||||
line = input.readSafeUtf8Line()
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} catch (ex: EOFException) {
|
||||
return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
|
||||
}
|
||||
}
|
||||
|
||||
var meta: Meta = Meta.EMPTY
|
||||
|
||||
if (line.startsWith(metaStart)) {
|
||||
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
|
||||
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
|
||||
meta = if (metaSize != null) {
|
||||
offset += metaSize.toUInt()
|
||||
metaFormat.readObject(input.readBinary(metaSize))
|
||||
} else {
|
||||
error("Can't partially read an envelope with undefined meta size")
|
||||
}
|
||||
}
|
||||
|
||||
do {
|
||||
line = input.readSafeUtf8Line() //?: return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
//returning an Envelope without data if end of input is reached
|
||||
} while (!line.startsWith(dataStart))
|
||||
|
||||
val dataSize = properties[DATA_LENGTH_PROPERTY]?.toULong()
|
||||
return PartialEnvelope(meta, offset, dataSize)
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
META_KEY put meta
|
||||
return Envelope(meta, data)
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
@ -183,45 +112,41 @@ public class TaglessEnvelopeFormat(
|
||||
|
||||
public const val TAGLESS_ENVELOPE_TYPE: String = "tagless"
|
||||
|
||||
public const val TAGLESS_ENVELOPE_HEADER: String = "#~DFTL~#"
|
||||
public const val META_START_PROPERTY: String = "metaSeparator"
|
||||
public const val DEFAULT_META_START: String = "#~META~#"
|
||||
public const val DATA_START_PROPERTY: String = "dataSeparator"
|
||||
public const val DEFAULT_DATA_START: String = "#~DATA~#"
|
||||
public val SEPARATOR_PREFIX: ByteString = "\n#~".encodeToByteString()
|
||||
|
||||
public val TAGLESS_ENVELOPE_HEADER: ByteString = "#~DFTL".encodeToByteString()
|
||||
|
||||
// public const val META_START_PROPERTY: String = "metaSeparator"
|
||||
public const val META_START: String = "#~META"
|
||||
|
||||
// public const val DATA_START_PROPERTY: String = "dataSeparator"
|
||||
public const val DATA_START: String = "#~DATA"
|
||||
|
||||
public const val END: String = "#~END"
|
||||
|
||||
public const val code: Int = 0x4446544c //DFTL
|
||||
|
||||
override val name: Name = TAGLESS_ENVELOPE_TYPE.asName()
|
||||
override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + TAGLESS_ENVELOPE_TYPE
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
||||
return TaglessEnvelopeFormat(context.io, meta)
|
||||
}
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat = TaglessEnvelopeFormat(context.io, meta)
|
||||
|
||||
private val default by lazy { invoke() }
|
||||
private val default by lazy { build(Global, Meta.EMPTY) }
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope =
|
||||
default.run { readPartial(input) }
|
||||
override fun readFrom(binary: Binary): Envelope = default.run { readFrom(binary) }
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
): Unit = default.run {
|
||||
writeEnvelope(
|
||||
output,
|
||||
envelope,
|
||||
metaFormatFactory,
|
||||
formatMeta
|
||||
)
|
||||
writeTo(sink, obj)
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope = default.readObject(input)
|
||||
override fun readFrom(source: Source): Envelope = default.readFrom(source)
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
|
||||
return try {
|
||||
binary.read {
|
||||
val string = readRawString(TAGLESS_ENVELOPE_HEADER.length)
|
||||
val string = readByteString(TAGLESS_ENVELOPE_HEADER.size)
|
||||
return@read if (string == TAGLESS_ENVELOPE_HEADER) {
|
||||
TaglessEnvelopeFormat(io)
|
||||
} else {
|
||||
|
@ -1,51 +1,235 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.charsets.Charsets
|
||||
import io.ktor.utils.io.charsets.decodeExactBytes
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.math.min
|
||||
|
||||
public fun Output.writeRawString(str: String) {
|
||||
writeFully(str.toByteArray(Charsets.ISO_8859_1))
|
||||
/**
|
||||
* Convert a string literal, containing only ASCII characters to a [ByteString].
|
||||
* Throws an error if there are non-ASCII characters.
|
||||
*/
|
||||
public fun String.toAsciiByteString(): ByteString {
|
||||
val bytes = ByteArray(length) {
|
||||
val char = get(it)
|
||||
val code = char.code
|
||||
if (code > Byte.MAX_VALUE) error("Symbol $char is not ASCII symbol") else code.toByte()
|
||||
}
|
||||
return ByteString(bytes)
|
||||
}
|
||||
|
||||
public fun Output.writeUtf8String(str: String) {
|
||||
writeFully(str.encodeToByteArray())
|
||||
}
|
||||
public inline fun Buffer(block: Sink.() -> Unit): Buffer = Buffer().apply(block)
|
||||
|
||||
@OptIn(ExperimentalIoApi::class)
|
||||
public fun Input.readRawString(size: Int): String {
|
||||
return Charsets.ISO_8859_1.newDecoder().decodeExactBytes(this, size)
|
||||
}
|
||||
//public fun Source.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found")
|
||||
|
||||
public fun Input.readUtf8String(): String = readBytes().decodeToString()
|
||||
public inline fun ByteArray(block: Sink.() -> Unit): ByteArray =
|
||||
Buffer(block).readByteArray()
|
||||
|
||||
public fun Input.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found")
|
||||
public inline fun Binary(block: Sink.() -> Unit): Binary =
|
||||
ByteArray(block).asBinary()
|
||||
|
||||
public inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): ByteArray {
|
||||
val builder = BytePacketBuilder(expectedSize)
|
||||
builder.block()
|
||||
return builder.build().readBytes()
|
||||
}
|
||||
|
||||
public inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
|
||||
buildByteArray(expectedSize, block).asBinary()
|
||||
public operator fun Binary.get(range: IntRange): Binary = view(range.first, range.last - range.first)
|
||||
|
||||
/**
|
||||
* View section of a [Binary] as an independent binary
|
||||
* Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts the binary, return null. If
|
||||
* multiple formats accept binary, throw an error.
|
||||
*/
|
||||
public class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary {
|
||||
|
||||
init {
|
||||
require(start > 0)
|
||||
require(start + size <= source.size) { "View boundary is outside source binary size" }
|
||||
public fun IOPlugin.peekBinaryEnvelopeFormat(binary: Binary): EnvelopeFormat? {
|
||||
val formats = envelopeFormatFactories.mapNotNull { factory ->
|
||||
factory.peekFormat(this@peekBinaryEnvelopeFormat, binary)
|
||||
}
|
||||
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
|
||||
return source.read(start + offset, min(size, atMost), block)
|
||||
return when (formats.size) {
|
||||
0 -> null
|
||||
1 -> formats.first()
|
||||
else -> error("Envelope format binary recognition clash: $formats")
|
||||
}
|
||||
}
|
||||
|
||||
public fun Binary.view(start: Int, size: Int): BinaryView = BinaryView(this, start, size)
|
||||
/**
|
||||
* A zero-copy read from
|
||||
*/
|
||||
@DFExperimental
|
||||
public fun IOPlugin.readEnvelope(
|
||||
binary: Binary,
|
||||
readNonEnvelopes: Boolean = false,
|
||||
formatPicker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryEnvelopeFormat,
|
||||
): Envelope = formatPicker(binary)?.readFrom(binary) ?: if (readNonEnvelopes) {
|
||||
// if no format accepts file, read it as binary
|
||||
Envelope(Meta.EMPTY, binary)
|
||||
} else error("Can't infer format for $binary")
|
||||
|
||||
public operator fun Binary.get(range: IntRange): BinaryView = view(range.first, range.last - range.first)
|
||||
@DFExperimental
|
||||
public fun IOPlugin.readEnvelope(
|
||||
string: String,
|
||||
readNonEnvelopes: Boolean = false,
|
||||
formatPicker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryEnvelopeFormat,
|
||||
): Envelope = readEnvelope(string.encodeToByteArray().asBinary(), readNonEnvelopes, formatPicker)
|
||||
|
||||
|
||||
private class RingByteArray(
|
||||
private val buffer: ByteArray,
|
||||
private var startIndex: Int = 0,
|
||||
var size: Int = 0,
|
||||
) {
|
||||
operator fun get(index: Int): Byte {
|
||||
require(index >= 0) { "Index must be positive" }
|
||||
require(index < size) { "Index $index is out of circular buffer size $size" }
|
||||
return buffer[startIndex.forward(index)]
|
||||
}
|
||||
|
||||
fun isFull(): Boolean = size == buffer.size
|
||||
|
||||
fun push(element: Byte) {
|
||||
buffer[startIndex.forward(size)] = element
|
||||
if (isFull()) startIndex++ else size++
|
||||
|
||||
}
|
||||
|
||||
private fun Int.forward(n: Int): Int = (this + n) % (buffer.size)
|
||||
|
||||
fun contentEquals(inputArray: ByteArray): Boolean = when {
|
||||
inputArray.size != buffer.size -> false
|
||||
size < buffer.size -> false
|
||||
else -> inputArray.indices.all { inputArray[it] == get(it) }
|
||||
}
|
||||
|
||||
fun contentEquals(byteString: ByteString): Boolean = when {
|
||||
byteString.size != buffer.size -> false
|
||||
size < buffer.size -> false
|
||||
else -> (0 until byteString.size).all { byteString[it] == get(it) }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private fun RingByteArray.toArray(): ByteArray = ByteArray(size) { get(it) }
|
||||
|
||||
/**
|
||||
* Read [Source] into [output] until designated multibyte [separator] and optionally continues until
|
||||
* the end of the line after it. Throw error if [separator] not found and [atMost] bytes are read.
|
||||
* Also fails if [separator] not found until the end of input.
|
||||
*
|
||||
* The Separator itself is not read into [Sink].
|
||||
*
|
||||
* @param errorOnEof if true error is thrown if separator is never encountered
|
||||
*
|
||||
* @return bytes actually being read, including separator
|
||||
*/
|
||||
public fun Source.readWithSeparatorTo(
|
||||
output: Sink?,
|
||||
separator: ByteString,
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int {
|
||||
var counter = 0
|
||||
val rb = RingByteArray(ByteArray(separator.size))
|
||||
|
||||
while (!exhausted()) {
|
||||
val byte = readByte()
|
||||
counter++
|
||||
if (counter >= atMost) error("Maximum number of bytes to be read $atMost reached.")
|
||||
rb.push(byte)
|
||||
if (rb.contentEquals(separator)) {
|
||||
return counter
|
||||
} else if (rb.isFull()) {
|
||||
output?.writeByte(rb[0])
|
||||
}
|
||||
}
|
||||
|
||||
if (errorOnEof) {
|
||||
error("Read to the end of input without encountering ${separator.decodeToString()}")
|
||||
} else {
|
||||
for (i in 1 until rb.size) {
|
||||
output?.writeByte(rb[i])
|
||||
}
|
||||
counter += (rb.size - 1)
|
||||
return counter
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discard all bytes until [separator] is encountered. Separator is discarded sa well.
|
||||
* Return the total number of bytes read.
|
||||
*/
|
||||
public fun Source.discardWithSeparator(
|
||||
separator: ByteString,
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int = readWithSeparatorTo(null, separator, atMost, errorOnEof)
|
||||
|
||||
/**
|
||||
* Discard all symbol until newline is discovered. Carriage return is not discarded.
|
||||
*/
|
||||
public fun Source.discardLine(
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int = discardWithSeparator("\n".encodeToByteString(), atMost, errorOnEof)
|
||||
|
||||
|
||||
/**
|
||||
* A [Source] based on [ByteArray]
|
||||
*/
|
||||
internal class ByteArraySource(
|
||||
private val byteArray: ByteArray,
|
||||
private val offset: Int = 0,
|
||||
private val size: Int = byteArray.size - offset,
|
||||
) : RawSource {
|
||||
|
||||
init {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset + size <= byteArray.size) { "End index is ${offset + size}, but the array size is ${byteArray.size}" }
|
||||
}
|
||||
|
||||
private var pointer = offset
|
||||
|
||||
override fun close() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
override fun readAtMostTo(sink: Buffer, byteCount: Long): Long {
|
||||
if (pointer == offset + size) return -1
|
||||
val byteRead = min(byteCount.toInt(), (size + offset - pointer))
|
||||
sink.write(byteArray, pointer, pointer + byteRead)
|
||||
pointer += byteRead
|
||||
return byteRead.toLong()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A [Source] based on [String]
|
||||
*/
|
||||
public class StringSource(
|
||||
public val string: String,
|
||||
public val offset: Int = 0,
|
||||
public val size: Int = string.length - offset,
|
||||
) : RawSource {
|
||||
|
||||
private var pointer = offset
|
||||
|
||||
override fun close() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
override fun readAtMostTo(sink: Buffer, byteCount: Long): Long {
|
||||
if (pointer == offset + size) return -1
|
||||
val byteRead = min(byteCount.toInt(), (size + offset - pointer))
|
||||
sink.writeString(string, pointer, pointer + byteRead)
|
||||
pointer += byteRead
|
||||
return byteRead.toLong()
|
||||
}
|
||||
}
|
||||
|
||||
public fun Sink.writeDouble(value: Double) {
|
||||
writeLong(value.toBits())
|
||||
}
|
||||
|
||||
public fun Source.readDouble(): Double = Double.fromBits(readLong())
|
||||
|
||||
public fun Sink.writeFloat(value: Float) {
|
||||
writeInt(value.toBits())
|
||||
}
|
||||
|
||||
public fun Source.readFloat(): Float = Float.fromBits(readInt())
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.readInt
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.readDouble
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlinx.io.writeString
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -9,42 +9,56 @@ import kotlin.test.assertEquals
|
||||
class EnvelopeFormatTest {
|
||||
val envelope = Envelope {
|
||||
type = "test.format"
|
||||
meta{
|
||||
meta {
|
||||
"d" put 22.2
|
||||
}
|
||||
data{
|
||||
writeDouble(22.2)
|
||||
// repeat(2000){
|
||||
// writeInt(it)
|
||||
// }
|
||||
data {
|
||||
writeString("12345678")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testTaggedFormat(){
|
||||
TaggedEnvelopeFormat.run {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
//println(byteArray.decodeToString())
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta,res.meta)
|
||||
val double = res.data?.read {
|
||||
readDouble()
|
||||
}
|
||||
assertEquals(22.2, double)
|
||||
fun testTaggedFormat() = with(TaggedEnvelopeFormat) {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testTaglessFormat(){
|
||||
TaglessEnvelopeFormat.run {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
//println(byteArray.decodeToString())
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta,res.meta)
|
||||
val double = res.data?.read {
|
||||
readDouble()
|
||||
}
|
||||
assertEquals(22.2, double)
|
||||
fun testTaglessFormat() = with(TaglessEnvelopeFormat) {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
println(byteArray.decodeToString())
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testManualDftl() {
|
||||
val envelopeString = """
|
||||
#~DFTL
|
||||
#~META
|
||||
{
|
||||
"@envelope": {
|
||||
"type": "test.format"
|
||||
},
|
||||
"d": 22.2
|
||||
}
|
||||
#~DATA
|
||||
12345678
|
||||
""".trimIndent()
|
||||
val res = TaglessEnvelopeFormat.readFromByteArray(envelopeString.encodeToByteArray())
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
}
|
@ -1,17 +1,58 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.readBytes
|
||||
import kotlinx.io.buffered
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlinx.io.readLine
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFails
|
||||
|
||||
class IOTest {
|
||||
@Test
|
||||
fun readBytes() {
|
||||
val bytes = ByteArray(8) { it.toByte() }
|
||||
val input = ByteReadPacket(bytes)
|
||||
@Suppress("UNUSED_VARIABLE") val first = input.readBytes(4)
|
||||
val second = input.readBytes(4)
|
||||
val input = ByteArraySource(bytes).buffered()
|
||||
@Suppress("UNUSED_VARIABLE") val first = input.readByteArray(4)
|
||||
val second = input.readByteArray(4)
|
||||
assertEquals(4.toByte(), second[0])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun readUntilSeparator() {
|
||||
val source = """
|
||||
aaa
|
||||
bbb
|
||||
---
|
||||
ccc
|
||||
ddd
|
||||
""".trimIndent()
|
||||
|
||||
val binary = source.encodeToByteArray().asBinary()
|
||||
|
||||
binary.read {
|
||||
val array = ByteArray {
|
||||
val read = readWithSeparatorTo(this, "---".encodeToByteString()) + discardLine()
|
||||
assertEquals(12, read)
|
||||
}
|
||||
assertEquals("""
|
||||
aaa
|
||||
bbb
|
||||
""".trimIndent(),array.decodeToString().trim())
|
||||
assertEquals("ccc", readLine()?.trim())
|
||||
}
|
||||
|
||||
assertFails {
|
||||
binary.read {
|
||||
discardWithSeparator("---".encodeToByteString(), atMost = 3 )
|
||||
}
|
||||
}
|
||||
|
||||
assertFails {
|
||||
binary.read{
|
||||
discardWithSeparator("-+-".encodeToByteString(), errorOnEof = true)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
@ -2,35 +2,19 @@ package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.serialization.json.*
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.JsonMeta.Companion.JSON_ARRAY_KEY
|
||||
import space.kscience.dataforge.values.ListValue
|
||||
import space.kscience.dataforge.values.number
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = buildByteArray {
|
||||
format.writeObject(this@buildByteArray, this@toByteArray)
|
||||
|
||||
fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = ByteArray {
|
||||
format.writeTo(this@ByteArray, this@toByteArray)
|
||||
}
|
||||
|
||||
fun MetaFormat.fromByteArray(packet: ByteArray): Meta {
|
||||
return packet.asBinary().read { readObject(this) }
|
||||
return packet.asBinary().read { readFrom(this) }
|
||||
}
|
||||
|
||||
class MetaFormatTest {
|
||||
@Test
|
||||
fun testBinaryMetaFormat() {
|
||||
val meta = Meta {
|
||||
"a" put 22
|
||||
"node" put {
|
||||
"b" put "DDD"
|
||||
"c" put 11.1
|
||||
"array" put doubleArrayOf(1.0, 2.0, 3.0)
|
||||
}
|
||||
}
|
||||
val bytes = meta.toByteArray(BinaryMetaFormat)
|
||||
val result = BinaryMetaFormat.fromByteArray(bytes)
|
||||
assertEquals(meta, result)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testJsonMetaFormat() {
|
||||
@ -51,36 +35,36 @@ class MetaFormatTest {
|
||||
if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}")
|
||||
}
|
||||
|
||||
assertEquals<Meta>(meta, result)
|
||||
assertEquals(meta, result)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testJsonToMeta() {
|
||||
val json = buildJsonArray {
|
||||
//top level array
|
||||
add(buildJsonArray {
|
||||
add(JsonPrimitive(88))
|
||||
add(buildJsonObject {
|
||||
addJsonArray {
|
||||
add(88)
|
||||
addJsonObject {
|
||||
put("c", "aasdad")
|
||||
put("d", true)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
add("value")
|
||||
add(buildJsonArray {
|
||||
add(JsonPrimitive(1.0))
|
||||
add(JsonPrimitive(2.0))
|
||||
add(JsonPrimitive(3.0))
|
||||
})
|
||||
addJsonArray {
|
||||
add(1.0)
|
||||
add(2.0)
|
||||
add(3.0)
|
||||
}
|
||||
}
|
||||
val meta = json.toMetaItem().node!!
|
||||
val meta = json.toMeta()
|
||||
|
||||
assertEquals(true, meta["$JSON_ARRAY_KEY[0].$JSON_ARRAY_KEY[1].d"].boolean)
|
||||
assertEquals("value", meta["$JSON_ARRAY_KEY[1]"].string)
|
||||
assertEquals(listOf(1.0, 2.0, 3.0), meta["$JSON_ARRAY_KEY[2"].value?.list?.map { it.number.toDouble() })
|
||||
assertEquals(true, meta["${Meta.JSON_ARRAY_KEY}[0].${Meta.JSON_ARRAY_KEY}[1].d"].boolean)
|
||||
assertEquals("value", meta["${Meta.JSON_ARRAY_KEY}[1]"].string)
|
||||
assertEquals(listOf(1.0, 2.0, 3.0), meta["${Meta.JSON_ARRAY_KEY}[2]"]?.value?.list?.map { it.double })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testJsonStringToMeta(){
|
||||
fun testJsonStringToMeta() {
|
||||
val jsonString = """
|
||||
{
|
||||
"comments": [
|
||||
@ -98,8 +82,8 @@ class MetaFormatTest {
|
||||
}
|
||||
""".trimIndent()
|
||||
val json = Json.parseToJsonElement(jsonString)
|
||||
val meta = json.toMetaItem().node!!
|
||||
assertEquals(ListValue.EMPTY, meta["comments"].value)
|
||||
val meta = json.toMeta()
|
||||
assertEquals(ListValue.EMPTY, meta["comments"]?.value)
|
||||
}
|
||||
|
||||
}
|
@ -5,9 +5,8 @@ import kotlinx.serialization.cbor.Cbor
|
||||
import kotlinx.serialization.json.Json
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaSerializer
|
||||
import space.kscience.dataforge.meta.TypedMetaItem
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -28,7 +27,7 @@ class MetaSerializerTest {
|
||||
fun testMetaSerialization() {
|
||||
val string = JSON_PRETTY.encodeToString(MetaSerializer, meta)
|
||||
println(string)
|
||||
val restored = JSON_PLAIN.decodeFromString(MetaSerializer, string)
|
||||
val restored = JSON_PLAIN.decodeFromString(MetaSerializer, string).seal()
|
||||
assertEquals(meta, restored)
|
||||
}
|
||||
|
||||
@ -43,7 +42,7 @@ class MetaSerializerTest {
|
||||
|
||||
@Test
|
||||
fun testNameSerialization() {
|
||||
val name = "a.b.c".toName()
|
||||
val name = Name.parse("a.b.c")
|
||||
val string = JSON_PRETTY.encodeToString(Name.serializer(), name)
|
||||
val restored = JSON_PLAIN.decodeFromString(Name.serializer(), string)
|
||||
assertEquals(name, restored)
|
||||
@ -52,7 +51,7 @@ class MetaSerializerTest {
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
@Test
|
||||
fun testMetaItemDescriptor() {
|
||||
val descriptor = TypedMetaItem.serializer(MetaSerializer).descriptor.getElementDescriptor(0)
|
||||
val descriptor = MetaSerializer.descriptor.getElementDescriptor(0)
|
||||
println(descriptor)
|
||||
}
|
||||
}
|
@ -1,5 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.writeString
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.int
|
||||
@ -18,32 +19,31 @@ class MultipartTest {
|
||||
"value" put it
|
||||
}
|
||||
data {
|
||||
writeUtf8String("Hello World $it")
|
||||
writeString("Hello World $it")
|
||||
repeat(300) {
|
||||
writeRawString("$it ")
|
||||
writeString("$it ")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val partsEnvelope = Envelope {
|
||||
envelopes(envelopes, TaglessEnvelopeFormat)
|
||||
envelopes(envelopes)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testParts() {
|
||||
TaglessEnvelopeFormat.run {
|
||||
val singleEnvelopeData = toBinary(envelopes[0])
|
||||
val singleEnvelopeSize = singleEnvelopeData.size
|
||||
val bytes = toBinary(partsEnvelope)
|
||||
assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
|
||||
val reconstructed = bytes.readWith(this)
|
||||
println(reconstructed.meta)
|
||||
val parts = reconstructed.parts()
|
||||
val envelope = parts[2].envelope(io)
|
||||
assertEquals(2, envelope.meta["value"].int)
|
||||
println(reconstructed.data!!.size)
|
||||
}
|
||||
val format = TaggedEnvelopeFormat
|
||||
val singleEnvelopeData = Binary(envelopes[0], format)
|
||||
val singleEnvelopeSize = singleEnvelopeData.size
|
||||
val bytes = Binary(partsEnvelope, format)
|
||||
assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
|
||||
val reconstructed = bytes.readWith(format)
|
||||
println(reconstructed.meta)
|
||||
val parts = reconstructed.parts()
|
||||
val envelope = parts[2].envelope()
|
||||
assertEquals(2, envelope.meta["value"].int)
|
||||
println(reconstructed.data!!.size)
|
||||
}
|
||||
|
||||
}
|
@ -1,12 +1,11 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.use
|
||||
import kotlinx.io.buffered
|
||||
|
||||
|
||||
fun <T : Any> IOFormat<T>.writeToByteArray(obj: T): ByteArray = buildByteArray {
|
||||
writeObject(this, obj)
|
||||
fun <T : Any> IOFormat<T>.writeToByteArray(obj: T): ByteArray = ByteArray {
|
||||
writeTo(this, obj)
|
||||
}
|
||||
fun <T : Any> IOFormat<T>.readFromByteArray(array: ByteArray): T = ByteReadPacket(array).use {
|
||||
readObject(it)
|
||||
fun <T : Any> IOFormat<T>.readFromByteArray(array: ByteArray): T = ByteArraySource(array).buffered().use {
|
||||
readFrom(it)
|
||||
}
|
@ -1,15 +1,18 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import io.ktor.utils.io.streams.asOutput
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asSink
|
||||
import kotlinx.io.buffered
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.isEmpty
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardOpenOption
|
||||
import kotlin.io.path.ExperimentalPathApi
|
||||
import kotlin.io.path.inputStream
|
||||
import kotlin.math.min
|
||||
import kotlin.reflect.full.isSupertypeOf
|
||||
@ -23,86 +26,114 @@ internal class PathBinary(
|
||||
override val size: Int = Files.size(path).toInt() - fileOffset,
|
||||
) : Binary {
|
||||
|
||||
@OptIn(ExperimentalPathApi::class)
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Source.() -> R): R = runBlocking {
|
||||
readSuspend(offset, atMost, block)
|
||||
}
|
||||
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R {
|
||||
val actualOffset = offset + fileOffset
|
||||
val actualSize = min(atMost, size - offset)
|
||||
val array = path.inputStream().use {
|
||||
it.skip(actualOffset.toLong())
|
||||
it.readNBytes(actualSize)
|
||||
}
|
||||
return ByteReadPacket(array).block()
|
||||
return ByteArraySource(array).buffered().use { it.block() }
|
||||
}
|
||||
|
||||
override fun view(offset: Int, binarySize: Int) = PathBinary(path, fileOffset + offset, binarySize)
|
||||
}
|
||||
|
||||
public fun Path.asBinary(): Binary = PathBinary(this)
|
||||
|
||||
public fun <R> Path.read(block: Input.() -> R): R = asBinary().read(block = block)
|
||||
public fun <R> Path.read(block: Source.() -> R): R = asBinary().read(block = block)
|
||||
|
||||
/**
|
||||
* Write a live output to a newly created file. If file does not exist, throws error
|
||||
*/
|
||||
public fun Path.write(block: Output.() -> Unit): Unit {
|
||||
public fun Path.write(block: Sink.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(this, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
|
||||
stream.asOutput().use(block)
|
||||
stream.asSink().buffered().use(block)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new file or append to exiting one with given output [block]
|
||||
*/
|
||||
public fun Path.append(block: Output.() -> Unit): Unit {
|
||||
public fun Path.append(block: Sink.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(
|
||||
this,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE
|
||||
)
|
||||
stream.asOutput().use(block)
|
||||
stream.asSink().buffered().use(block)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new file or replace existing one using given output [block]
|
||||
*/
|
||||
public fun Path.rewrite(block: Output.() -> Unit): Unit {
|
||||
public fun Path.rewrite(block: Sink.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(
|
||||
this,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE
|
||||
)
|
||||
stream.asOutput().use(block)
|
||||
stream.asSink().buffered().use(block)
|
||||
}
|
||||
|
||||
public fun Path.readEnvelope(format: EnvelopeFormat): Envelope {
|
||||
val partialEnvelope: PartialEnvelope = asBinary().read {
|
||||
format.run {
|
||||
readPartial(this@read)
|
||||
}
|
||||
}
|
||||
val offset: Int = partialEnvelope.dataOffset.toInt()
|
||||
val size: Int = partialEnvelope.dataSize?.toInt() ?: (Files.size(this).toInt() - offset)
|
||||
val binary = PathBinary(this, offset, size)
|
||||
return SimpleEnvelope(partialEnvelope.meta, binary)
|
||||
}
|
||||
@DFExperimental
|
||||
public fun EnvelopeFormat.readFile(path: Path): Envelope = readFrom(path.asBinary())
|
||||
|
||||
/**
|
||||
* Resolve IOFormat based on type
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
|
||||
return ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>?
|
||||
public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? =
|
||||
ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>?
|
||||
|
||||
|
||||
public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
|
||||
public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
|
||||
|
||||
/**
|
||||
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
|
||||
* If [path] is a directory search for file starting with `meta` in it.
|
||||
*
|
||||
* Returns null if meta could not be resolved
|
||||
*/
|
||||
public fun IOPlugin.readMetaFileOrNull(
|
||||
path: Path,
|
||||
formatOverride: MetaFormat? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
): Meta? {
|
||||
if (!Files.exists(path)) return null
|
||||
|
||||
val actualPath: Path = if (Files.isDirectory(path)) {
|
||||
Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) }
|
||||
?: return null
|
||||
} else {
|
||||
path
|
||||
}
|
||||
val extension = actualPath.fileName.toString().substringAfterLast('.')
|
||||
|
||||
val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: return null
|
||||
return actualPath.read {
|
||||
metaFormat.readMeta(this, descriptor)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
|
||||
* If [path] is a directory search for file starting with `meta` in it
|
||||
* If [path] is a directory search for file starting with `meta` in it.
|
||||
*
|
||||
* Fails if nothing works.
|
||||
*/
|
||||
public fun IOPlugin.readMetaFile(
|
||||
path: Path,
|
||||
formatOverride: MetaFormat? = null,
|
||||
descriptor: NodeDescriptor? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
): Meta {
|
||||
if (!Files.exists(path)) error("Meta file $path does not exist")
|
||||
|
||||
val actualPath: Path = if (Files.isDirectory(path)) {
|
||||
Files.list(path).asSequence().find { it.fileName.startsWith("meta") }
|
||||
Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) }
|
||||
?: error("The directory $path does not contain meta file")
|
||||
} else {
|
||||
path
|
||||
@ -110,13 +141,12 @@ public fun IOPlugin.readMetaFile(
|
||||
val extension = actualPath.fileName.toString().substringAfterLast('.')
|
||||
|
||||
val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: error("Can't resolve meta format $extension")
|
||||
return metaFormat.run {
|
||||
actualPath.read {
|
||||
readMeta(this, descriptor)
|
||||
}
|
||||
return actualPath.read {
|
||||
metaFormat.readMeta(this, descriptor)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Write meta to file using [metaFormat]. If [path] is a directory, write a file with name equals name of [metaFormat].
|
||||
* Like "meta.json"
|
||||
@ -125,7 +155,7 @@ public fun IOPlugin.writeMetaFile(
|
||||
path: Path,
|
||||
meta: Meta,
|
||||
metaFormat: MetaFormatFactory = JsonMetaFormat,
|
||||
descriptor: NodeDescriptor? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
) {
|
||||
val actualPath = if (Files.isDirectory(path)) {
|
||||
path.resolve("@" + metaFormat.name.toString())
|
||||
@ -145,19 +175,9 @@ public fun IOPlugin.writeMetaFile(
|
||||
*/
|
||||
public fun IOPlugin.peekFileEnvelopeFormat(path: Path): EnvelopeFormat? {
|
||||
val binary = path.asBinary()
|
||||
val formats = envelopeFormatFactories.mapNotNull { factory ->
|
||||
factory.peekFormat(this@peekFileEnvelopeFormat, binary)
|
||||
}
|
||||
|
||||
return when (formats.size) {
|
||||
0 -> null
|
||||
1 -> formats.first()
|
||||
else -> error("Envelope format binary recognition clash: $formats")
|
||||
}
|
||||
return peekBinaryEnvelopeFormat(binary)
|
||||
}
|
||||
|
||||
public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
|
||||
public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
|
||||
|
||||
/**
|
||||
* Read and envelope from file if the file exists, return null if file does not exist.
|
||||
@ -203,22 +223,11 @@ public fun IOPlugin.readEnvelopeFile(
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
return formatPicker(path)?.let { format ->
|
||||
path.readEnvelope(format)
|
||||
} ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
|
||||
return formatPicker(path)?.readFile(path) ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
|
||||
SimpleEnvelope(Meta.EMPTY, path.asBinary())
|
||||
} else error("Can't infer format for file $path")
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a binary into file. Throws an error if file already exists
|
||||
*/
|
||||
public fun <T : Any> IOFormat<T>.writeToFile(path: Path, obj: T) {
|
||||
path.write {
|
||||
writeObject(this, obj)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write envelope file to given [path] using [envelopeFormat] and optional [metaFormat]
|
||||
*/
|
||||
@ -227,10 +236,9 @@ public fun IOPlugin.writeEnvelopeFile(
|
||||
path: Path,
|
||||
envelope: Envelope,
|
||||
envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,
|
||||
metaFormat: MetaFormatFactory? = null,
|
||||
) {
|
||||
path.rewrite {
|
||||
envelopeFormat.writeEnvelope(this, envelope, metaFormat ?: envelopeFormat.defaultMetaFormat)
|
||||
envelopeFormat.writeTo(this, envelope)
|
||||
}
|
||||
}
|
||||
|
||||
@ -255,7 +263,7 @@ public fun IOPlugin.writeEnvelopeDirectory(
|
||||
val dataFile = path.resolve(IOPlugin.DATA_FILE_NAME)
|
||||
dataFile.write {
|
||||
envelope.data?.read {
|
||||
val copied = copyTo(this@write)
|
||||
val copied = transferTo(this@write)
|
||||
if (copied != envelope.data?.size?.toLong()) {
|
||||
error("The number of copied bytes does not equal data size")
|
||||
}
|
||||
|
@ -0,0 +1,11 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asSource
|
||||
import kotlinx.io.buffered
|
||||
|
||||
|
||||
public fun IOPlugin.resource(name: String): Binary? = { }.javaClass.getResource(name)?.readBytes()?.asBinary()
|
||||
|
||||
public inline fun <R> IOPlugin.readResource(name: String, block: Source.() -> R): R =
|
||||
{ }.javaClass.getResource(name)?.openStream()?.asSource()?.buffered()?.block() ?: error("Can't read resource $name")
|
@ -0,0 +1,27 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import space.kscience.dataforge.context.ContextBuilder
|
||||
import space.kscience.dataforge.meta.set
|
||||
import space.kscience.dataforge.meta.string
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.Path
|
||||
|
||||
|
||||
public val IOPlugin.workDirectory: Path
|
||||
get() {
|
||||
val workDirectoryPath = meta[IOPlugin.WORK_DIRECTORY_KEY].string
|
||||
?: context.properties[IOPlugin.WORK_DIRECTORY_KEY].string
|
||||
?: ".dataforge"
|
||||
|
||||
return Path(workDirectoryPath)
|
||||
}
|
||||
|
||||
public fun ContextBuilder.workDirectory(path: String) {
|
||||
properties {
|
||||
set(IOPlugin.WORK_DIRECTORY_KEY, path)
|
||||
}
|
||||
}
|
||||
|
||||
public fun ContextBuilder.workDirectory(path: Path) {
|
||||
workDirectory(path.toAbsolutePath().toString())
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
|
23
dataforge-meta/README.md
Normal file
23
dataforge-meta/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-meta
|
||||
|
||||
Meta definition and basic operations on meta
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-meta:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-meta:0.7.0")
|
||||
}
|
||||
```
|
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user