Compare commits
149 Commits
Author | SHA1 | Date | |
---|---|---|---|
c754dc3471 | |||
8f7d754301 | |||
c923c3e7d3 | |||
99b2d941c8 | |||
191af77f57 | |||
79759c5256 | |||
2eb965e563 | |||
1b29e377ca | |||
2634a19285 | |||
261c415d3d | |||
e52d509c2b | |||
706521a6b6 | |||
94000689da | |||
851fdda311 | |||
cbbcd18df3 | |||
dc2bf5da83 | |||
f732b85cc5 | |||
259b882e63 | |||
526f230300 | |||
|
7fca5db390 | ||
|
be160ba98a | ||
|
182f206b88 | ||
3806f97c77 | |||
d5ebef404f | |||
3644533043 | |||
ee5afcdafe | |||
de476fb273 | |||
a136db16ff | |||
a699c36f8e | |||
2aba1b48dc | |||
cfa20eedba | |||
|
f78f0f814f | ||
4e7ead0763 | |||
4543648cda | |||
b6949310ea | |||
2c2f33427a | |||
29fa30fb51 | |||
f3afb5e9fe | |||
61c8df9eb0 | |||
707b59e6fc | |||
add400b324 | |||
58c5355e25 | |||
f83b759e75 | |||
7d88f828d7 | |||
5d7ddb4e00 | |||
82838b6a92 | |||
e41fdfc086 | |||
4117a05df4 | |||
5406a6a64c | |||
0cc4dc0db7 | |||
233639f0b6 | |||
70bd92f019 | |||
f8eea45ed0 | |||
e14c0a695e | |||
3c1fe23366 | |||
81e2ad06cc | |||
6ca76cff17 | |||
9a24e1e392 | |||
9cceb44a90 | |||
0b68c1edae | |||
b8869570ce | |||
4833128857 | |||
6bd8a7acbc | |||
f5d32ba511 | |||
0fc2198832 | |||
a546552540 | |||
fe92e8fccf | |||
7d9189e15c | |||
0622bacc4d | |||
bedab0dc86 | |||
f0820a3bed | |||
665f317e4e | |||
82d37f4b55 | |||
6d396368b7 | |||
77857289f0 | |||
eaa9d40d60 | |||
6b41163ed3 | |||
e5000171f1 | |||
3c6bc15716 | |||
11143e4ba1 | |||
91621864c2 | |||
1e97165328 | |||
|
be8e971436 | ||
|
9cc30b1f4e | ||
|
7414e60192 | ||
|
8c0bc05a9a | ||
c480cd8e4d | |||
64e0c554cc | |||
532e0c253b | |||
c423dc214e | |||
d178c4ff0d | |||
|
387ab8747e | ||
3f54eee578 | |||
aded38254e | |||
00d964eef3 | |||
b07d281a83 | |||
81cdd38c40 | |||
0ad6852e36 | |||
a71bb732da | |||
|
acfe9c2f74 | ||
ce8074c104 | |||
922a3b07ee | |||
81abbe28a9 | |||
da9d6e7639 | |||
66c708d9fb | |||
c1065c2885 | |||
e5f422f9ca | |||
c01bc36d41 | |||
be2daca25e | |||
b968d735ce | |||
67554a8c98 | |||
24187722e4 | |||
d3c129526d | |||
5632487dca | |||
e432b07201 | |||
28a6914747 | |||
90a92c4121 | |||
b404615145 | |||
7aec2f3547 | |||
3ba5a9076b | |||
8763d63e28 | |||
4a76063093 | |||
5fbbac465a | |||
b387b21554 | |||
c8bd3390cb | |||
a3479e74f7 | |||
9f5b010847 | |||
679175391a | |||
14455c2b2b | |||
a9cec666a3 | |||
d2ea1a975e | |||
73b3bbe7fc | |||
474597777c | |||
9d3c7149b7 | |||
bc9cd3b5a8 | |||
254163bdef | |||
|
3bdaf332cb | ||
|
6e769d1089 | ||
|
e62ff61814 | ||
1037c45c0d | |||
82b328f797 | |||
a6f1e54255 | |||
3888b2d9e7 | |||
0acb6ec448 | |||
|
b616e3ad6d | ||
|
d1381cc98c | ||
|
543023d2df | ||
|
d769b0d389 | ||
|
3fc698dd09 |
21
.github/workflows/build.yml
vendored
21
.github/workflows/build.yml
vendored
@ -1,6 +1,9 @@
|
||||
name: Gradle build
|
||||
|
||||
on: [ push ]
|
||||
on:
|
||||
push:
|
||||
branches: [ dev, master ]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@ -8,20 +11,22 @@ jobs:
|
||||
matrix:
|
||||
os: [ macOS-latest, windows-latest ]
|
||||
runs-on: ${{matrix.os}}
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- name: Checkout the repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
uses: DeLaGuardo/setup-graalvm@4.0
|
||||
with:
|
||||
java-version: 11
|
||||
- name: Add msys to path
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: SETX PATH "%PATH%;C:\msys64\mingw64\bin"
|
||||
graalvm: 21.2.0
|
||||
java: java11
|
||||
arch: amd64
|
||||
- name: Cache gradle
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.gradle/caches
|
||||
path: |
|
||||
~/.gradle/caches
|
||||
~/.gradle/wrapper
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-gradle-
|
||||
@ -33,4 +38,4 @@ jobs:
|
||||
restore-keys: |
|
||||
${{ runner.os }}-gradle-
|
||||
- name: Build
|
||||
run: ./gradlew build --no-daemon --stacktrace
|
||||
run: ./gradlew build --build-cache --no-daemon --stacktrace
|
||||
|
36
.github/workflows/pages.yml
vendored
36
.github/workflows/pages.yml
vendored
@ -1,33 +1,31 @@
|
||||
name: Dokka publication
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
release:
|
||||
types: [ created ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- name: Checkout the repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
- uses: actions/checkout@v3.0.0
|
||||
- uses: actions/setup-java@v3.0.0
|
||||
with:
|
||||
java-version: 11
|
||||
- name: Cache gradle
|
||||
uses: actions/cache@v2
|
||||
distribution: liberica
|
||||
- name: Cache konan
|
||||
uses: actions/cache@v3.0.1
|
||||
with:
|
||||
path: ~/.gradle/caches
|
||||
key: ubuntu-20.04-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
path: ~/.konan
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
restore-keys: |
|
||||
ubuntu-20.04-gradle-
|
||||
- name: Build
|
||||
run: |
|
||||
./gradlew dokkaHtmlMultiModule --no-daemon --no-parallel --stacktrace
|
||||
mv build/dokka/htmlMultiModule/-modules.html build/dokka/htmlMultiModule/index.html
|
||||
- name: Deploy to GitHub Pages
|
||||
uses: JamesIves/github-pages-deploy-action@4.1.0
|
||||
${{ runner.os }}-gradle-
|
||||
- uses: gradle/gradle-build-action@v2.4.2
|
||||
with:
|
||||
arguments: dokkaHtmlMultiModule --no-parallel
|
||||
- uses: JamesIves/github-pages-deploy-action@v4.3.0
|
||||
with:
|
||||
branch: gh-pages
|
||||
folder: build/dokka/htmlMultiModule
|
||||
|
57
.github/workflows/publish.yml
vendored
57
.github/workflows/publish.yml
vendored
@ -3,8 +3,7 @@ name: Gradle publish
|
||||
on:
|
||||
workflow_dispatch:
|
||||
release:
|
||||
types:
|
||||
- created
|
||||
types: [ created ]
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
@ -12,27 +11,16 @@ jobs:
|
||||
name: publish
|
||||
strategy:
|
||||
matrix:
|
||||
os: [macOS-latest, windows-latest]
|
||||
os: [ macOS-latest, windows-latest ]
|
||||
runs-on: ${{matrix.os}}
|
||||
steps:
|
||||
- name: Checkout the repo
|
||||
uses: actions/checkout@v2
|
||||
- name: Set up JDK 11
|
||||
uses: actions/setup-java@v1
|
||||
- uses: actions/checkout@v3.0.0
|
||||
- uses: actions/setup-java@v3.10.0
|
||||
with:
|
||||
java-version: 11
|
||||
- name: Add msys to path
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: SETX PATH "%PATH%;C:\msys64\mingw64\bin"
|
||||
- name: Cache gradle
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.gradle/caches
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-gradle-
|
||||
distribution: liberica
|
||||
- name: Cache konan
|
||||
uses: actions/cache@v2
|
||||
uses: actions/cache@v3.0.1
|
||||
with:
|
||||
path: ~/.konan
|
||||
key: ${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts') }}
|
||||
@ -40,20 +28,23 @@ jobs:
|
||||
${{ runner.os }}-gradle-
|
||||
- name: Publish Windows Artifacts
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: >
|
||||
./gradlew release --no-daemon
|
||||
-Ppublishing.enabled=true
|
||||
-Ppublishing.github.user=${{ secrets.PUBLISHING_GITHUB_USER }}
|
||||
-Ppublishing.github.token=${{ secrets.PUBLISHING_GITHUB_TOKEN }}
|
||||
-Ppublishing.space.user=${{ secrets.PUBLISHING_SPACE_USER }}
|
||||
-Ppublishing.space.token=${{ secrets.PUBLISHING_SPACE_TOKEN }}
|
||||
uses: gradle/gradle-build-action@v2.4.2
|
||||
with:
|
||||
arguments: |
|
||||
publishAllPublicationsToSpaceRepository
|
||||
-Ppublishing.targets=all
|
||||
-Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
|
||||
-Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}
|
||||
- name: Publish Mac Artifacts
|
||||
if: matrix.os == 'macOS-latest'
|
||||
run: >
|
||||
./gradlew release --no-daemon
|
||||
-Ppublishing.enabled=true
|
||||
-Ppublishing.platform=macosX64
|
||||
-Ppublishing.github.user=${{ secrets.PUBLISHING_GITHUB_USER }}
|
||||
-Ppublishing.github.token=${{ secrets.PUBLISHING_GITHUB_TOKEN }}
|
||||
-Ppublishing.space.user=${{ secrets.PUBLISHING_SPACE_USER }}
|
||||
-Ppublishing.space.token=${{ secrets.PUBLISHING_SPACE_TOKEN }}
|
||||
uses: gradle/gradle-build-action@v2.4.2
|
||||
with:
|
||||
arguments: |
|
||||
publishMacosX64PublicationToSpaceRepository
|
||||
publishMacosArm64PublicationToSpaceRepository
|
||||
publishIosX64PublicationToSpaceRepository
|
||||
publishIosArm64PublicationToSpaceRepository
|
||||
publishIosSimulatorArm64PublicationToSpaceRepository
|
||||
-Ppublishing.targets=all
|
||||
-Ppublishing.space.user=${{ secrets.SPACE_APP_ID }}
|
||||
-Ppublishing.space.token=${{ secrets.SPACE_APP_SECRET }}
|
||||
|
152
CHANGELOG.md
152
CHANGELOG.md
@ -1,6 +1,7 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
## Unreleased
|
||||
|
||||
### Added
|
||||
|
||||
### Changed
|
||||
@ -12,14 +13,136 @@
|
||||
### Fixed
|
||||
|
||||
### Security
|
||||
## [0.4.0]
|
||||
|
||||
## 0.7.0 - 2023-11-26
|
||||
|
||||
### Added
|
||||
|
||||
- Obligatory `type: KType` and `descriptor` property for `MetaConverters`
|
||||
- Added separate `Meta`, `SealedMeta` and `ObservableMutableMeta` builders.
|
||||
|
||||
### Changed
|
||||
|
||||
- Meta converter `metaToObject` returns a non-nullable type. Additional method `metaToObjectOrNull` for nullable return.
|
||||
- Kotlin 1.9.20.
|
||||
- Migrated from ktor-io to kotlinx-io.
|
||||
- `MutableMeta` builder now returns a simplified version of meta that does not hold listeners.
|
||||
- More concise names for read/write methods in IO.
|
||||
- Remove unnecessary confusion with `get`/`getMeta` by removing `getMeta` from the interface.
|
||||
|
||||
### Deprecated
|
||||
|
||||
- `String.parseValue` is replaced with `Value.parse`
|
||||
|
||||
### Fixed
|
||||
|
||||
- Memory leak in SealedMeta builder
|
||||
|
||||
## 0.6.2 - 2023-07-29
|
||||
|
||||
### Changed
|
||||
|
||||
- Meta to Json serializer now serializes a single item with index as an array. It is important for plotly integration.
|
||||
- Meta to Json serializes Meta without children a value as literal or array instead of an object with `@value` field.
|
||||
|
||||
## 0.6.1 - 2023-03-31
|
||||
|
||||
### Added
|
||||
|
||||
- File cache for workspace
|
||||
- Smart task metadata transformation for workspace
|
||||
- Add `readOnly` property to descriptors
|
||||
- Add `specOrNull` delegate to meta and Scheme
|
||||
- Suspended read methods to the `Binary`
|
||||
- Synchronously accessed `meta` to all `DataSet`s
|
||||
- More fine-grained types in Action builders.
|
||||
|
||||
### Changed
|
||||
|
||||
- `Name::replaceLast` API
|
||||
- `PluginFactory` no longer requires plugin class
|
||||
- Collection<Named> toMap -> associateByName
|
||||
- Simplified `DFTL` envelope format. Closing symbols are unnecessary. Properties are discontinued.
|
||||
- Meta `get` method allows nullable receiver
|
||||
- `withDefault` functions do not add new keys to meta children and are consistent.
|
||||
- `dataforge.meta.values` package is merged into `dataforge.meta` for better star imports
|
||||
- Kotlin 1.8.20
|
||||
- `Factory` is now `fun interface` and uses `build` instead of `invoke`. `invoke moved to an extension.
|
||||
- KTor 2.0
|
||||
- DataTree `items` call is blocking.
|
||||
- DataSet `getData` is no longer suspended and renamed to `get`
|
||||
- DataSet operates with sequences of data instead of flows
|
||||
- PartialEnvelope uses `Int` instead `UInt`.
|
||||
- `ActiveDataSet` renamed to `DataSource`
|
||||
- `selectOne`->`getByType`
|
||||
- Data traversal in `DataSet` is done via iterator
|
||||
- Remove all unnecessary properties for `IOFormat`
|
||||
- Separate interfaces for `IOReader` and `IOWriter`
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Context.fetch -> Context.request
|
||||
|
||||
### Fixed
|
||||
|
||||
- `readDataDirectory` does not split names with dots
|
||||
- Front matter reader does not crash on non-UTF files
|
||||
- Meta file name in readMeta from directory
|
||||
- Tagless and FrontMatter envelope partial readers fix.
|
||||
|
||||
## 0.5.2
|
||||
|
||||
### Added
|
||||
|
||||
- Yaml plugin
|
||||
- Partial fix to #53
|
||||
|
||||
### Fixed
|
||||
|
||||
- MutableMetaImpl attachment and checks
|
||||
- Listeners in observable meta are replaced by lists
|
||||
- JS number comparison bug.
|
||||
|
||||
## 0.5.0
|
||||
|
||||
### Added
|
||||
|
||||
- Experimental `listOfSpec` delegate.
|
||||
|
||||
### Changed
|
||||
|
||||
- **API breaking** Config is deprecated, use `ObservableMeta` instead.
|
||||
- **API breaking** Descriptor no has a member property `defaultValue` instead of `defaultItem()` extension. It caches default value state on the first call. It is done because computing default on each call is too expensive.
|
||||
- Kotlin 1.5.10
|
||||
- Build tools 0.10.0
|
||||
- Relaxed type restriction on `MetaConverter`. Now nullables are available.
|
||||
- **Huge API-breaking refactoring of Meta**. Meta now can have both value and children. There is only one kind of descriptor now.
|
||||
- **API breaking** `String.toName()` is replaced by `Name.parse()`
|
||||
- **API breaking** Configurable`config` changed to `meta`
|
||||
|
||||
### Removed
|
||||
|
||||
- `Config`
|
||||
- Public PluginManager mutability
|
||||
- Tables and tables-exposed moved to the separate project `tables.kt`
|
||||
- BinaryMetaFormat. Use CBOR encoding instead
|
||||
|
||||
### Fixed
|
||||
|
||||
- Proper json array index treatment.
|
||||
- Proper json index for single-value array.
|
||||
|
||||
## 0.4.0
|
||||
|
||||
### Added
|
||||
|
||||
- LogManager plugin
|
||||
- dataforge-context API dependency on SLF4j
|
||||
- Context `withEnv` and `fetch` methods to manipulate plugins without changing plugins after creation.
|
||||
- Split `ItemDescriptor` into builder and read-only part
|
||||
|
||||
### Changed
|
||||
|
||||
- Kotlin-logging moved from common to JVM and JS. Replaced by console for native.
|
||||
- Package changed to `space.kscience`
|
||||
- Scheme made observable
|
||||
@ -29,19 +152,22 @@
|
||||
- Refactor loggers
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Direct use of PluginManager
|
||||
|
||||
### Removed
|
||||
|
||||
- Common dependency on Kotlin-logging
|
||||
- Kotlinx-io fork dependency. Replaced by Ktor-io.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Scheme properties properly handle children property change.
|
||||
|
||||
### Security
|
||||
## 0.3.0
|
||||
|
||||
## [0.3.0]
|
||||
### Added
|
||||
|
||||
- Yaml meta format based on yaml.kt
|
||||
- `Path` builders
|
||||
- Special ValueType for lists
|
||||
@ -49,6 +175,7 @@
|
||||
- Multiplatform yaml meta
|
||||
|
||||
### Changed
|
||||
|
||||
- `ListValue` and `DoubleArrayValue` implement `Iterable`.
|
||||
- Changed the logic of `Value::isList` to check for type instead of size
|
||||
- `Meta{}` builder made inline
|
||||
@ -62,18 +189,10 @@
|
||||
- \[Major breaking change\] Full refactor of DataTree/DataSource
|
||||
- \[Major Breaking change\] Replace KClass with KType in data. Remove direct access to constructors with types.
|
||||
|
||||
### Deprecated
|
||||
|
||||
### Removed
|
||||
|
||||
### Fixed
|
||||
|
||||
### Security
|
||||
|
||||
## [0.2.0]
|
||||
### Added
|
||||
## 0.2.0
|
||||
|
||||
### Changed
|
||||
|
||||
- Context content resolution refactor
|
||||
- Kotlin 1.4.10 (build tools 0.6.0)
|
||||
- Empty query in Name is null instead of ""
|
||||
@ -83,15 +202,16 @@
|
||||
- Configurable is no longer MutableItemProvider. All functionality moved to Scheme.
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Context activation API
|
||||
- TextRenderer
|
||||
|
||||
### Removed
|
||||
|
||||
- Functional server prototype
|
||||
- `dataforge-output` module
|
||||
|
||||
### Fixed
|
||||
|
||||
- Global context CoroutineScope resolution
|
||||
- Library mode compliance
|
||||
|
||||
### Security
|
||||
|
201
LICENSE
Normal file
201
LICENSE
Normal file
@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
39
README.md
39
README.md
@ -3,53 +3,36 @@
|
||||
|
||||
![Gradle build](https://github.com/mipt-npm/dataforge-core/workflows/Gradle%20build/badge.svg)
|
||||
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-context](dataforge-context)
|
||||
>
|
||||
### [dataforge-context](dataforge-context)
|
||||
> Context and provider definitions
|
||||
>
|
||||
> **Maturity**: DEVELOPMENT
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-data](dataforge-data)
|
||||
>
|
||||
### [dataforge-data](dataforge-data)
|
||||
>
|
||||
> **Maturity**: EXPERIMENTAL
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-io](dataforge-io)
|
||||
>
|
||||
### [dataforge-io](dataforge-io)
|
||||
> IO module
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
> **Maturity**: EXPERIMENTAL
|
||||
|
||||
* ### [dataforge-meta](dataforge-meta)
|
||||
>
|
||||
### [dataforge-meta](dataforge-meta)
|
||||
> Meta definition and basic operations on meta
|
||||
>
|
||||
> **Maturity**: DEVELOPMENT
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-scripting](dataforge-scripting)
|
||||
>
|
||||
### [dataforge-scripting](dataforge-scripting)
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-tables](dataforge-tables)
|
||||
>
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-workspace](dataforge-workspace)
|
||||
>
|
||||
### [dataforge-workspace](dataforge-workspace)
|
||||
>
|
||||
> **Maturity**: EXPERIMENTAL
|
||||
<hr/>
|
||||
|
||||
* ### [dataforge-io-yaml](dataforge-io/dataforge-io-yaml)
|
||||
### [dataforge-io/dataforge-io-yaml](dataforge-io/dataforge-io-yaml)
|
||||
> YAML meta converters and Front Matter envelope format
|
||||
>
|
||||
> **Maturity**: PROTOTYPE
|
||||
<hr/>
|
||||
|
||||
|
@ -1,33 +1,39 @@
|
||||
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
||||
import space.kscience.gradle.useApache2Licence
|
||||
import space.kscience.gradle.useSPCTeam
|
||||
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.project")
|
||||
id("space.kscience.gradle.project")
|
||||
}
|
||||
|
||||
allprojects {
|
||||
group = "space.kscience"
|
||||
version = "0.4.0"
|
||||
version = "0.7.0"
|
||||
}
|
||||
|
||||
subprojects {
|
||||
apply(plugin = "maven-publish")
|
||||
|
||||
tasks.withType<KotlinCompile> {
|
||||
kotlinOptions {
|
||||
freeCompilerArgs = freeCompilerArgs + "-Xcontext-receivers"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
readme {
|
||||
readmeTemplate = file("docs/templates/README-TEMPLATE.md")
|
||||
}
|
||||
|
||||
changelog{
|
||||
version = project.version.toString()
|
||||
}
|
||||
|
||||
ksciencePublish {
|
||||
github("dataforge-core")
|
||||
space("https://maven.pkg.jetbrains.space/mipt-npm/p/sci/maven")
|
||||
pom("https://github.com/SciProgCentre/kmath") {
|
||||
useApache2Licence()
|
||||
useSPCTeam()
|
||||
}
|
||||
repository("spc","https://maven.sciprog.center/kscience")
|
||||
sonatype()
|
||||
}
|
||||
|
||||
apiValidation {
|
||||
if(project.version.toString().contains("dev")) {
|
||||
validationDisabled = true
|
||||
}
|
||||
nonPublicMarkers.add("space.kscience.dataforge.misc.DFExperimental")
|
||||
}
|
23
dataforge-context/README.md
Normal file
23
dataforge-context/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-context
|
||||
|
||||
Context and provider definitions
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-context:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-context:0.7.0")
|
||||
}
|
||||
```
|
@ -3,16 +3,17 @@ public abstract class space/kscience/dataforge/context/AbstractPlugin : space/ks
|
||||
public fun <init> (Lspace/kscience/dataforge/meta/Meta;)V
|
||||
public synthetic fun <init> (Lspace/kscience/dataforge/meta/Meta;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
|
||||
public fun attach (Lspace/kscience/dataforge/context/Context;)V
|
||||
public final fun dependsOn ()Ljava/util/Map;
|
||||
public fun dependsOn ()Ljava/util/Map;
|
||||
public fun detach ()V
|
||||
public fun getContext ()Lspace/kscience/dataforge/context/Context;
|
||||
public fun getMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
protected final fun require (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)Lkotlin/properties/ReadOnlyProperty;
|
||||
public static synthetic fun require$default (Lspace/kscience/dataforge/context/AbstractPlugin;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty;
|
||||
public fun isAttached ()Z
|
||||
protected final fun require (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/meta/Meta;)Lkotlin/properties/ReadOnlyProperty;
|
||||
public static synthetic fun require$default (Lspace/kscience/dataforge/context/AbstractPlugin;Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)Lkotlin/properties/ReadOnlyProperty;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/AbstractPluginKt {
|
||||
public static final fun toMap (Ljava/util/Collection;)Ljava/util/Map;
|
||||
public static final fun associateByName (Ljava/util/Collection;)Ljava/util/Map;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/ClassLoaderPlugin : space/kscience/dataforge/context/AbstractPlugin {
|
||||
@ -33,8 +34,8 @@ public final class space/kscience/dataforge/context/ClassLoaderPluginKt {
|
||||
public class space/kscience/dataforge/context/Context : kotlinx/coroutines/CoroutineScope, space/kscience/dataforge/meta/MetaRepr, space/kscience/dataforge/misc/Named, space/kscience/dataforge/provider/Provider {
|
||||
public static final field Companion Lspace/kscience/dataforge/context/Context$Companion;
|
||||
public static final field PROPERTY_TARGET Ljava/lang/String;
|
||||
public final fun buildContext (Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
|
||||
public static synthetic fun buildContext$default (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
|
||||
public final fun buildContext (Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
|
||||
public static synthetic fun buildContext$default (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/names/Name;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Context;
|
||||
public fun close ()V
|
||||
public fun content (Ljava/lang/String;)Ljava/util/Map;
|
||||
public final fun content (Ljava/lang/String;Z)Ljava/util/Map;
|
||||
@ -57,7 +58,6 @@ public abstract interface class space/kscience/dataforge/context/ContextAware {
|
||||
public final class space/kscience/dataforge/context/ContextBuilder {
|
||||
public final fun build ()Lspace/kscience/dataforge/context/Context;
|
||||
public final fun getName ()Lspace/kscience/dataforge/names/Name;
|
||||
public final fun name (Ljava/lang/String;)V
|
||||
public final fun plugin (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)V
|
||||
public final fun plugin (Lspace/kscience/dataforge/context/Plugin;)V
|
||||
public final fun plugin (Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;)V
|
||||
@ -67,11 +67,9 @@ public final class space/kscience/dataforge/context/ContextBuilder {
|
||||
public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginFactory;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
|
||||
public static synthetic fun plugin$default (Lspace/kscience/dataforge/context/ContextBuilder;Lspace/kscience/dataforge/context/PluginTag;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
|
||||
public final fun properties (Lkotlin/jvm/functions/Function1;)V
|
||||
public final fun setName (Lspace/kscience/dataforge/names/Name;)V
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/ContextBuilderKt {
|
||||
public static final fun withEnv (Lspace/kscience/dataforge/context/Context;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Context;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/DefaultLogManager : space/kscience/dataforge/context/AbstractPlugin, space/kscience/dataforge/context/LogManager {
|
||||
@ -83,14 +81,17 @@ public final class space/kscience/dataforge/context/DefaultLogManager : space/ks
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/DefaultLogManager$Companion : space/kscience/dataforge/context/PluginFactory {
|
||||
public synthetic fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
|
||||
public fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/context/DefaultLogManager;
|
||||
public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public fun getType ()Lkotlin/reflect/KClass;
|
||||
public synthetic fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
|
||||
public fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Lspace/kscience/dataforge/context/DefaultLogManager;
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/context/Factory {
|
||||
public abstract fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
|
||||
public abstract fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/FactoryKt {
|
||||
public static final fun invoke (Lspace/kscience/dataforge/context/Factory;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
|
||||
public static synthetic fun invoke$default (Lspace/kscience/dataforge/context/Factory;Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;ILjava/lang/Object;)Ljava/lang/Object;
|
||||
}
|
||||
|
||||
@ -145,6 +146,7 @@ public abstract interface class space/kscience/dataforge/context/Plugin : space/
|
||||
public abstract fun getMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
public fun getName ()Lspace/kscience/dataforge/names/Name;
|
||||
public abstract fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public abstract fun isAttached ()Z
|
||||
public fun toMeta ()Lspace/kscience/dataforge/meta/Meta;
|
||||
}
|
||||
|
||||
@ -152,11 +154,26 @@ public final class space/kscience/dataforge/context/Plugin$Companion {
|
||||
public static final field TARGET Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginBuilder {
|
||||
public fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
|
||||
public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
|
||||
public final fun build ()Lspace/kscience/dataforge/context/PluginFactory;
|
||||
public final fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public final fun provides (Ljava/lang/String;Ljava/util/Map;)V
|
||||
public final fun provides (Ljava/lang/String;[Lspace/kscience/dataforge/misc/Named;)V
|
||||
public final fun requires (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;)V
|
||||
public static synthetic fun requires$default (Lspace/kscience/dataforge/context/PluginBuilder;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ILjava/lang/Object;)V
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginBuilderKt {
|
||||
public static final fun PluginFactory (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/PluginFactory;
|
||||
public static synthetic fun PluginFactory$default (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/PluginFactory;
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/context/PluginFactory : space/kscience/dataforge/context/Factory {
|
||||
public static final field Companion Lspace/kscience/dataforge/context/PluginFactory$Companion;
|
||||
public static final field TYPE Ljava/lang/String;
|
||||
public abstract fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public abstract fun getType ()Lkotlin/reflect/KClass;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginFactory$Companion {
|
||||
@ -164,21 +181,15 @@ public final class space/kscience/dataforge/context/PluginFactory$Companion {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginManager : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker, space/kscience/dataforge/context/ContextAware {
|
||||
public fun <init> (Lspace/kscience/dataforge/context/Context;)V
|
||||
public final fun fetch (Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;Z)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun fetch (Lspace/kscience/dataforge/context/PluginFactory;ZLkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun fetch$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginFactory;Lspace/kscience/dataforge/meta/Meta;ZILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun fetch$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginFactory;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun find (ZLkotlin/jvm/functions/Function1;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun find$default (Lspace/kscience/dataforge/context/PluginManager;ZLkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun get (Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;Z)Ljava/lang/Object;
|
||||
public final fun get (Lspace/kscience/dataforge/context/PluginTag;Z)Lspace/kscience/dataforge/context/Plugin;
|
||||
public static synthetic fun get$default (Lspace/kscience/dataforge/context/PluginManager;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Ljava/lang/Object;
|
||||
public static synthetic fun get$default (Lspace/kscience/dataforge/context/PluginManager;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Lspace/kscience/dataforge/context/Plugin;
|
||||
public final fun getByType (Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;Z)Ljava/lang/Object;
|
||||
public static synthetic fun getByType$default (Lspace/kscience/dataforge/context/PluginManager;Lkotlin/reflect/KClass;Lspace/kscience/dataforge/context/PluginTag;ZILjava/lang/Object;)Ljava/lang/Object;
|
||||
public fun getContext ()Lspace/kscience/dataforge/context/Context;
|
||||
public fun iterator ()Ljava/util/Iterator;
|
||||
public final fun list (Z)Ljava/util/Collection;
|
||||
public final fun remove (Lspace/kscience/dataforge/context/Plugin;)V
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginTag : space/kscience/dataforge/meta/MetaRepr {
|
||||
@ -201,14 +212,25 @@ public final class space/kscience/dataforge/context/PluginTag : space/kscience/d
|
||||
public fun toString ()Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginTag$$serializer : kotlinx/serialization/internal/GeneratedSerializer {
|
||||
public static final field INSTANCE Lspace/kscience/dataforge/context/PluginTag$$serializer;
|
||||
public fun childSerializers ()[Lkotlinx/serialization/KSerializer;
|
||||
public synthetic fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Ljava/lang/Object;
|
||||
public fun deserialize (Lkotlinx/serialization/encoding/Decoder;)Lspace/kscience/dataforge/context/PluginTag;
|
||||
public fun getDescriptor ()Lkotlinx/serialization/descriptors/SerialDescriptor;
|
||||
public synthetic fun serialize (Lkotlinx/serialization/encoding/Encoder;Ljava/lang/Object;)V
|
||||
public fun serialize (Lkotlinx/serialization/encoding/Encoder;Lspace/kscience/dataforge/context/PluginTag;)V
|
||||
public fun typeParametersSerializers ()[Lkotlinx/serialization/KSerializer;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/PluginTag$Companion {
|
||||
public final fun fromString (Ljava/lang/String;)Lspace/kscience/dataforge/context/PluginTag;
|
||||
public final fun serializer ()Lkotlinx/serialization/KSerializer;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/ResolveKt {
|
||||
public static final fun gather (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;Z)Ljava/util/Map;
|
||||
public static synthetic fun gather$default (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;ZILjava/lang/Object;)Ljava/util/Map;
|
||||
public static synthetic fun gatherInSequence$default (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lkotlin/reflect/KClass;ZILjava/lang/Object;)Lkotlin/sequences/Sequence;
|
||||
public static final fun getValues (Lkotlin/sequences/Sequence;)Lkotlin/sequences/Sequence;
|
||||
public static final fun resolve (Lspace/kscience/dataforge/context/Context;Ljava/lang/String;Lspace/kscience/dataforge/names/Name;Lkotlin/reflect/KClass;)Ljava/lang/Object;
|
||||
}
|
||||
@ -222,32 +244,9 @@ public final class space/kscience/dataforge/context/SlfLogManager : space/kscien
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/context/SlfLogManager$Companion : space/kscience/dataforge/context/PluginFactory {
|
||||
public synthetic fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Ljava/lang/Object;
|
||||
public fun build (Lspace/kscience/dataforge/context/Context;Lspace/kscience/dataforge/meta/Meta;)Lspace/kscience/dataforge/context/SlfLogManager;
|
||||
public fun getTag ()Lspace/kscience/dataforge/context/PluginTag;
|
||||
public fun getType ()Lkotlin/reflect/KClass;
|
||||
public synthetic fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Ljava/lang/Object;
|
||||
public fun invoke (Lspace/kscience/dataforge/meta/Meta;Lspace/kscience/dataforge/context/Context;)Lspace/kscience/dataforge/context/SlfLogManager;
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/Attribute : java/lang/annotation/Annotation {
|
||||
public abstract fun key ()Ljava/lang/String;
|
||||
public abstract fun value ()Ljava/lang/String;
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/Attributes : java/lang/annotation/Annotation {
|
||||
public abstract fun attrs ()[Lspace/kscience/dataforge/descriptors/Attribute;
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/ItemDef : java/lang/annotation/Annotation {
|
||||
public abstract fun info ()Ljava/lang/String;
|
||||
public abstract fun multiple ()Z
|
||||
public abstract fun required ()Z
|
||||
}
|
||||
|
||||
public abstract interface annotation class space/kscience/dataforge/descriptors/ValueDef : java/lang/annotation/Annotation {
|
||||
public abstract fun allowed ()[Ljava/lang/String;
|
||||
public abstract fun def ()Ljava/lang/String;
|
||||
public abstract fun enumeration ()Ljava/lang/Class;
|
||||
public abstract fun type ()[Lspace/kscience/dataforge/values/ValueType;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/properties/PropertyKt {
|
||||
@ -257,7 +256,6 @@ public final class space/kscience/dataforge/properties/SchemePropertyKt {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/DfTypeKt {
|
||||
public static final fun getDfType (Lkotlin/reflect/KClass;)Ljava/lang/String;
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/Path : java/lang/Iterable, kotlin/jvm/internal/markers/KMappedMarker {
|
||||
@ -311,7 +309,8 @@ public final class space/kscience/dataforge/provider/PathToken {
|
||||
}
|
||||
|
||||
public final class space/kscience/dataforge/provider/PathToken$Companion {
|
||||
public final fun parse (Ljava/lang/String;)Lspace/kscience/dataforge/provider/PathToken;
|
||||
public final fun parse (Ljava/lang/String;Z)Lspace/kscience/dataforge/provider/PathToken;
|
||||
public static synthetic fun parse$default (Lspace/kscience/dataforge/provider/PathToken$Companion;Ljava/lang/String;ZILjava/lang/Object;)Lspace/kscience/dataforge/provider/PathToken;
|
||||
}
|
||||
|
||||
public abstract interface class space/kscience/dataforge/provider/Provider {
|
||||
|
@ -1,35 +1,24 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
description = "Context and provider definitions"
|
||||
|
||||
kscience {
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
useCoroutines()
|
||||
}
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
val commonMain by getting{
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
}
|
||||
}
|
||||
jvmMain {
|
||||
dependencies {
|
||||
api(kotlin("reflect"))
|
||||
api("org.slf4j:slf4j-api:1.7.30")
|
||||
}
|
||||
}
|
||||
jsMain {
|
||||
dependencies {
|
||||
|
||||
}
|
||||
}
|
||||
useSerialization()
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
}
|
||||
dependencies(jvmMain){
|
||||
api(kotlin("reflect"))
|
||||
api("org.slf4j:slf4j-api:1.7.30")
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.DEVELOPMENT
|
||||
readme {
|
||||
maturity = space.kscience.gradle.Maturity.DEVELOPMENT
|
||||
}
|
@ -1,6 +1,7 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.properties.ReadOnlyProperty
|
||||
@ -11,6 +12,8 @@ public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plu
|
||||
private var _context: Context? = null
|
||||
private val dependencies = HashMap<PluginFactory<*>, Meta>()
|
||||
|
||||
override val isAttached: Boolean get() = _context != null
|
||||
|
||||
override val context: Context
|
||||
get() = _context ?: error("Plugin $tag is not attached")
|
||||
|
||||
@ -22,21 +25,33 @@ public abstract class AbstractPlugin(override val meta: Meta = Meta.EMPTY) : Plu
|
||||
this._context = null
|
||||
}
|
||||
|
||||
final override fun dependsOn(): Map<PluginFactory<*>, Meta> = dependencies
|
||||
override fun dependsOn(): Map<PluginFactory<*>, Meta> = dependencies
|
||||
|
||||
protected fun <P : Plugin> require(
|
||||
factory: PluginFactory<P>,
|
||||
type: KClass<P>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): ReadOnlyProperty<AbstractPlugin, P> {
|
||||
dependencies[factory] = meta
|
||||
return PluginDependencyDelegate(factory, type)
|
||||
}
|
||||
|
||||
/**
|
||||
* Register plugin dependency and return a delegate which provides lazily initialized reference to dependent plugin
|
||||
*/
|
||||
protected fun <P : Plugin> require(factory: PluginFactory<P>, meta: Meta = Meta.EMPTY): ReadOnlyProperty<AbstractPlugin, P> {
|
||||
dependencies[factory] = meta
|
||||
return PluginDependencyDelegate(factory.type)
|
||||
}
|
||||
protected inline fun <reified P : Plugin> require(
|
||||
factory: PluginFactory<P>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): ReadOnlyProperty<AbstractPlugin, P> = require(factory, P::class, meta)
|
||||
}
|
||||
|
||||
public fun <T : Named> Collection<T>.toMap(): Map<Name, T> = associate { it.name to it }
|
||||
public fun <T : Named> Collection<T>.associateByName(): Map<Name, T> = associate { it.name to it }
|
||||
|
||||
private class PluginDependencyDelegate<P : Plugin>(val type: KClass<out P>) : ReadOnlyProperty<AbstractPlugin, P> {
|
||||
private class PluginDependencyDelegate<P : Plugin>(val factory: PluginFactory<P>, val type: KClass<P>) :
|
||||
ReadOnlyProperty<AbstractPlugin, P> {
|
||||
@OptIn(DFInternal::class)
|
||||
override fun getValue(thisRef: AbstractPlugin, property: KProperty<*>): P {
|
||||
return thisRef.context.plugins[type] ?: error("Plugin with type $type not found")
|
||||
if (!thisRef.isAttached) error("Plugin dependency must not be called eagerly during initialization.")
|
||||
return thisRef.context.plugins.getByType(type, factory.tag) ?: error("Plugin ${factory.tag} not found")
|
||||
}
|
||||
}
|
@ -3,15 +3,13 @@ package space.kscience.dataforge.context
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import space.kscience.dataforge.meta.Laminate
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.meta.itemSequence
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.misc.ThreadSafe
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.provider.Provider
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.jvm.Synchronized
|
||||
|
||||
/**
|
||||
* The local environment for anything being done in DataForge framework. Contexts are organized into tree structure with [Global] at the top.
|
||||
@ -26,6 +24,7 @@ import kotlin.jvm.Synchronized
|
||||
public open class Context internal constructor(
|
||||
final override val name: Name,
|
||||
public val parent: Context?,
|
||||
plugins: Set<Plugin>, // set of unattached plugins
|
||||
meta: Meta,
|
||||
) : Named, MetaRepr, Provider, CoroutineScope {
|
||||
|
||||
@ -42,20 +41,20 @@ public open class Context internal constructor(
|
||||
/**
|
||||
* A [PluginManager] for current context
|
||||
*/
|
||||
public val plugins: PluginManager by lazy { PluginManager(this) }
|
||||
public val plugins: PluginManager by lazy { PluginManager(this, plugins) }
|
||||
|
||||
override val defaultTarget: String get() = Plugin.TARGET
|
||||
|
||||
public fun content(target: String, inherit: Boolean): Map<Name, Any> {
|
||||
return if (inherit) {
|
||||
when (target) {
|
||||
PROPERTY_TARGET -> properties.itemSequence().toMap()
|
||||
PROPERTY_TARGET -> properties.nodeSequence().toMap()
|
||||
Plugin.TARGET -> plugins.list(true).associateBy { it.name }
|
||||
else -> emptyMap()
|
||||
}
|
||||
} else {
|
||||
when (target) {
|
||||
PROPERTY_TARGET -> properties.layers.firstOrNull()?.itemSequence()?.toMap() ?: emptyMap()
|
||||
PROPERTY_TARGET -> properties.layers.firstOrNull()?.nodeSequence()?.toMap() ?: emptyMap()
|
||||
Plugin.TARGET -> plugins.list(false).associateBy { it.name }
|
||||
else -> emptyMap()
|
||||
}
|
||||
@ -73,16 +72,16 @@ public open class Context internal constructor(
|
||||
private val childrenContexts = HashMap<Name, Context>()
|
||||
|
||||
/**
|
||||
* Build and register a child context
|
||||
* Get and validate existing context or build and register a new child context.
|
||||
* @param name the relative (tail) name of the new context. If null, uses context hash code as a marker.
|
||||
*/
|
||||
@Synchronized
|
||||
public fun buildContext(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context {
|
||||
val newContext = ContextBuilder(this)
|
||||
.apply { name?.let { name(it) } }
|
||||
.apply(block)
|
||||
.build()
|
||||
childrenContexts[newContext.name] = newContext
|
||||
return newContext
|
||||
@OptIn(DFExperimental::class)
|
||||
@ThreadSafe
|
||||
public fun buildContext(name: Name? = null, block: ContextBuilder.() -> Unit = {}): Context {
|
||||
val existing = name?.let { childrenContexts[name] }
|
||||
return existing?.modify(block) ?: ContextBuilder(this, name).apply(block).build().also {
|
||||
childrenContexts[it.name] = it
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -97,8 +96,8 @@ public open class Context internal constructor(
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
"parent" to parent?.name
|
||||
"properties" put properties.layers.firstOrNull()
|
||||
"plugins" put plugins.map { it.toMeta() }
|
||||
properties.layers.firstOrNull()?.let { set("properties", it) }
|
||||
"plugins" putIndexed plugins.map { it.toMeta() }
|
||||
}
|
||||
|
||||
public companion object {
|
||||
|
@ -1,14 +1,15 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFBuilder
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.NameToken
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.plus
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.collections.set
|
||||
@ -19,39 +20,35 @@ import kotlin.collections.set
|
||||
@DFBuilder
|
||||
public class ContextBuilder internal constructor(
|
||||
private val parent: Context,
|
||||
public var name: Name? = null,
|
||||
public val name: Name? = null,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
) {
|
||||
internal val factories = HashMap<PluginFactory<*>, Meta>()
|
||||
internal var meta = meta.toMutableMeta()
|
||||
|
||||
public fun properties(action: MetaBuilder.() -> Unit) {
|
||||
public fun properties(action: MutableMeta.() -> Unit) {
|
||||
meta.action()
|
||||
}
|
||||
|
||||
public fun name(string: String) {
|
||||
this.name = string.toName()
|
||||
}
|
||||
|
||||
@OptIn(DFExperimental::class)
|
||||
private fun findPluginFactory(tag: PluginTag): PluginFactory<*> =
|
||||
parent.gatherInSequence<PluginFactory<*>>(PluginFactory.TYPE).values
|
||||
.find { it.tag.matches(tag) } ?: error("Can't resolve plugin factory for $tag")
|
||||
|
||||
public fun plugin(tag: PluginTag, metaBuilder: MetaBuilder.() -> Unit = {}) {
|
||||
public fun plugin(tag: PluginTag, mutableMeta: MutableMeta.() -> Unit = {}) {
|
||||
val factory = findPluginFactory(tag)
|
||||
factories[factory] = Meta(metaBuilder)
|
||||
factories[factory] = Meta(mutableMeta)
|
||||
}
|
||||
|
||||
public fun plugin(factory: PluginFactory<*>, meta: Meta) {
|
||||
factories[factory] = meta
|
||||
}
|
||||
|
||||
public fun plugin(factory: PluginFactory<*>, metaBuilder: MetaBuilder.() -> Unit = {}) {
|
||||
factories[factory] = Meta(metaBuilder)
|
||||
public fun plugin(factory: PluginFactory<*>, mutableMeta: MutableMeta.() -> Unit = {}) {
|
||||
factories[factory] = Meta(mutableMeta)
|
||||
}
|
||||
|
||||
public fun plugin(name: String, group: String = "", version: String = "", action: MetaBuilder.() -> Unit = {}) {
|
||||
public fun plugin(name: String, group: String = "", version: String = "", action: MutableMeta.() -> Unit = {}) {
|
||||
plugin(PluginTag(name, group, version), action)
|
||||
}
|
||||
|
||||
@ -63,30 +60,52 @@ public class ContextBuilder internal constructor(
|
||||
}
|
||||
|
||||
public fun build(): Context {
|
||||
val contextName = name ?: "@auto[${hashCode().toUInt().toString(16)}]".toName()
|
||||
return Context(contextName, parent, meta.seal()).apply {
|
||||
factories.forEach { (factory, meta) ->
|
||||
@Suppress("DEPRECATION")
|
||||
plugins.load(factory, meta)
|
||||
val contextName = name ?: NameToken("@auto",hashCode().toUInt().toString(16)).asName()
|
||||
val plugins = HashMap<PluginTag, Plugin>()
|
||||
|
||||
fun addPlugin(factory: PluginFactory<*>, meta: Meta) {
|
||||
val existing = plugins[factory.tag]
|
||||
// Add if does not exist
|
||||
if (existing == null) {
|
||||
//TODO bypass if parent already has plugin with given meta?
|
||||
val plugin = factory.build(parent, meta)
|
||||
|
||||
for ((depFactory, deoMeta) in plugin.dependsOn()) {
|
||||
addPlugin(depFactory, deoMeta)
|
||||
}
|
||||
|
||||
parent.logger.info { "Loading plugin ${plugin.name} into $contextName" }
|
||||
plugins[plugin.tag] = plugin
|
||||
} else if (existing.meta != meta) {
|
||||
error("Plugin with tag ${factory.tag} and meta $meta already exists in $contextName")
|
||||
}
|
||||
//bypass if exists with the same meta
|
||||
}
|
||||
|
||||
factories.forEach { (factory, meta) ->
|
||||
addPlugin(factory, meta)
|
||||
}
|
||||
|
||||
return Context(contextName, parent, plugins.values.toSet(), meta.seal())
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if current context contains all plugins required by the builder and return it it does or forks to a new context
|
||||
* Check if current context contains all plugins required by the builder and return it does or forks to a new context
|
||||
* if it does not.
|
||||
*/
|
||||
public fun Context.withEnv(block: ContextBuilder.() -> Unit): Context {
|
||||
@DFExperimental
|
||||
public fun Context.modify(block: ContextBuilder.() -> Unit): Context {
|
||||
|
||||
fun Context.contains(factory: PluginFactory<*>, meta: Meta): Boolean {
|
||||
val loaded = plugins[factory.tag] ?: return false
|
||||
return loaded.meta == meta
|
||||
}
|
||||
|
||||
val builder = ContextBuilder(this, name + "env", properties).apply(block)
|
||||
val builder = ContextBuilder(this, name + "mod", properties).apply(block)
|
||||
val requiresFork = builder.factories.any { (factory, meta) ->
|
||||
!contains(factory, meta)
|
||||
} || ((properties as Meta) == builder.meta)
|
||||
|
||||
return if (requiresFork) builder.build() else this
|
||||
}
|
@ -2,6 +2,11 @@ package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
|
||||
public interface Factory<out T : Any> {
|
||||
public operator fun invoke(meta: Meta = Meta.EMPTY, context: Context = Global): T
|
||||
}
|
||||
public fun interface Factory<out T> {
|
||||
public fun build(context: Context, meta: Meta): T
|
||||
}
|
||||
|
||||
public operator fun <T> Factory<T>.invoke(
|
||||
meta: Meta = Meta.EMPTY,
|
||||
context: Context = Global,
|
||||
): T = build(context, meta)
|
@ -1,23 +1,24 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import kotlinx.coroutines.GlobalScope
|
||||
import kotlinx.coroutines.CoroutineName
|
||||
import kotlinx.coroutines.Job
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.native.concurrent.ThreadLocal
|
||||
|
||||
internal expect val globalLoggerFactory: PluginFactory<out LogManager>
|
||||
internal expect fun getGlobalLoggerFactory(): PluginFactory<out LogManager>
|
||||
|
||||
/**
|
||||
* A global root context. Closing [Global] terminates the framework.
|
||||
*/
|
||||
@ThreadLocal
|
||||
private object GlobalContext : Context("GLOBAL".asName(), null, Meta.EMPTY) {
|
||||
override val coroutineContext: CoroutineContext = GlobalScope.coroutineContext + Job()
|
||||
private object GlobalContext : Context("GLOBAL".asName(), null, emptySet(), Meta.EMPTY) {
|
||||
override val coroutineContext: CoroutineContext = Job() + CoroutineName("GlobalContext")
|
||||
}
|
||||
|
||||
public val Global: Context get() = GlobalContext
|
||||
|
||||
public fun Context(name: String? = null, block: ContextBuilder.() -> Unit = {}): Context =
|
||||
Global.buildContext(name, block)
|
||||
Global.buildContext(name?.parseAsName(), block)
|
@ -4,7 +4,6 @@ import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
public fun interface Logger {
|
||||
public fun log(tag: String, body: () -> String)
|
||||
@ -63,10 +62,9 @@ public class DefaultLogManager : AbstractPlugin(), LogManager {
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
public companion object : PluginFactory<DefaultLogManager> {
|
||||
override fun invoke(meta: Meta, context: Context): DefaultLogManager = DefaultLogManager()
|
||||
override fun build(context: Context, meta: Meta): DefaultLogManager = DefaultLogManager()
|
||||
|
||||
override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.default")
|
||||
override val type: KClass<out DefaultLogManager> = DefaultLogManager::class
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,7 +73,7 @@ public class DefaultLogManager : AbstractPlugin(), LogManager {
|
||||
*/
|
||||
public val Context.logger: LogManager
|
||||
get() = plugins.find(inherit = true) { it is LogManager } as? LogManager
|
||||
?: globalLoggerFactory(context = Global).apply { attach(Global) }
|
||||
?: getGlobalLoggerFactory().build(context = Global, meta = Meta.EMPTY).apply { attach(Global) }
|
||||
|
||||
/**
|
||||
* The named proxy logger for a context member
|
||||
|
@ -3,10 +3,10 @@ package space.kscience.dataforge.context
|
||||
import space.kscience.dataforge.context.Plugin.Companion.TARGET
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import space.kscience.dataforge.provider.Provider
|
||||
|
||||
/**
|
||||
@ -18,7 +18,7 @@ import space.kscience.dataforge.provider.Provider
|
||||
*
|
||||
* create - configure - attach - detach - destroy
|
||||
*/
|
||||
@Type(TARGET)
|
||||
@DfId(TARGET)
|
||||
public interface Plugin : Named, ContextAware, Provider, MetaRepr {
|
||||
|
||||
/**
|
||||
@ -31,7 +31,7 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
|
||||
/**
|
||||
* The name of this plugin ignoring version and group
|
||||
*/
|
||||
override val name: Name get() = tag.name.toName()
|
||||
override val name: Name get() = tag.name.parseAsName()
|
||||
|
||||
/**
|
||||
* Plugin dependencies which are required to attach this plugin. Plugin
|
||||
@ -53,6 +53,8 @@ public interface Plugin : Named, ContextAware, Provider, MetaRepr {
|
||||
*/
|
||||
public fun detach()
|
||||
|
||||
public val isAttached: Boolean
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
"context" put context.name.toString()
|
||||
"type" to this::class.simpleName
|
||||
|
@ -0,0 +1,57 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
|
||||
/**
|
||||
* A convenience factory to build simple plugins
|
||||
*/
|
||||
public class PluginBuilder(
|
||||
name: String,
|
||||
group: String = "",
|
||||
version: String = "",
|
||||
) {
|
||||
public val tag: PluginTag = PluginTag(name, group, version)
|
||||
|
||||
private val content = HashMap<String, MutableMap<Name, Any>>()
|
||||
private val dependencies = HashMap<PluginFactory<*>, Meta>()
|
||||
|
||||
public fun requires(
|
||||
factory: PluginFactory<*>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
) {
|
||||
dependencies[factory] = meta
|
||||
}
|
||||
|
||||
public fun provides(target: String, items: Map<Name, Any>) {
|
||||
content.getOrPut(target) { HashMap() }.putAll(items)
|
||||
}
|
||||
|
||||
public fun provides(target: String, vararg items: Named) {
|
||||
provides(target, items.associateBy { it.name })
|
||||
}
|
||||
|
||||
public fun build(): PluginFactory<*> {
|
||||
|
||||
return object : PluginFactory<Plugin> {
|
||||
override val tag: PluginTag get() = this@PluginBuilder.tag
|
||||
|
||||
override fun build(context: Context, meta: Meta): Plugin = object : AbstractPlugin() {
|
||||
override val tag: PluginTag get() = this@PluginBuilder.tag
|
||||
|
||||
override fun content(target: String): Map<Name, Any> = this@PluginBuilder.content[target] ?: emptyMap()
|
||||
|
||||
override fun dependsOn(): Map<PluginFactory<*>, Meta> = this@PluginBuilder.dependencies
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public fun PluginFactory(
|
||||
name: String,
|
||||
group: String = "",
|
||||
version: String = "",
|
||||
block: PluginBuilder.() -> Unit,
|
||||
): PluginFactory<*> = PluginBuilder(name, group, version).apply(block).build()
|
@ -1,13 +1,11 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import kotlin.reflect.KClass
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
|
||||
@Type(PluginFactory.TYPE)
|
||||
@DfId(PluginFactory.TYPE)
|
||||
public interface PluginFactory<T : Plugin> : Factory<T> {
|
||||
public val tag: PluginTag
|
||||
public val type: KClass<out T>
|
||||
|
||||
public companion object {
|
||||
public const val TYPE: String = "pluginFactory"
|
||||
@ -18,7 +16,6 @@ public interface PluginFactory<T : Plugin> : Factory<T> {
|
||||
* Plugin factory created for the specific actual plugin
|
||||
*/
|
||||
internal class DeFactoPluginFactory<T : Plugin>(val plugin: T) : PluginFactory<T> {
|
||||
override fun invoke(meta: Meta, context: Context): T = plugin
|
||||
override fun build(context: Context, meta: Meta): T = plugin
|
||||
override val tag: PluginTag get() = plugin.tag
|
||||
override val type: KClass<out T> get() = plugin::class
|
||||
}
|
||||
|
@ -1,8 +1,10 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.plus
|
||||
import kotlin.reflect.KClass
|
||||
import kotlin.reflect.cast
|
||||
|
||||
|
||||
/**
|
||||
@ -11,12 +13,10 @@ import kotlin.reflect.KClass
|
||||
* @property context A context for this plugin manager
|
||||
* @author Alexander Nozik
|
||||
*/
|
||||
public class PluginManager(override val context: Context) : ContextAware, Iterable<Plugin> {
|
||||
|
||||
/**
|
||||
* A set of loaded plugins
|
||||
*/
|
||||
private val plugins: HashSet<Plugin> = HashSet()
|
||||
public class PluginManager internal constructor(
|
||||
override val context: Context,
|
||||
private val plugins: Set<Plugin>,
|
||||
) : ContextAware, Iterable<Plugin> {
|
||||
|
||||
init {
|
||||
plugins.forEach { it.attach(context) }
|
||||
@ -66,92 +66,35 @@ public class PluginManager(override val context: Context) : ContextAware, Iterab
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
public operator fun <T : Any> get(type: KClass<out T>, tag: PluginTag? = null, recursive: Boolean = true): T? =
|
||||
find(recursive) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) } as T?
|
||||
@DFInternal
|
||||
public fun <T : Any> getByType(type: KClass<T>, tag: PluginTag? = null, inherit: Boolean = true): T? =
|
||||
find(inherit) { type.isInstance(it) && (tag == null || tag.matches(it.tag)) }?.let { type.cast(it) }
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline operator fun <reified T : Any> get(tag: PluginTag? = null, recursive: Boolean = true): T? =
|
||||
get(T::class, tag, recursive)
|
||||
getByType(T::class, tag, recursive)
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline operator fun <reified T : Plugin> get(factory: PluginFactory<T>, recursive: Boolean = true): T? =
|
||||
get(factory.type, factory.tag, recursive)
|
||||
|
||||
/**
|
||||
* Load given plugin into this manager and return loaded instance.
|
||||
* Throw error if plugin of the same type and tag already exists in manager.
|
||||
*
|
||||
* @param plugin
|
||||
* @return
|
||||
*/
|
||||
@Deprecated("Use immutable contexts instead")
|
||||
private fun <T : Plugin> load(plugin: T): T {
|
||||
if (get(plugin::class, plugin.tag, recursive = false) != null) {
|
||||
error("Plugin with tag ${plugin.tag} already exists in ${context.name}")
|
||||
} else {
|
||||
for ((factory, meta) in plugin.dependsOn()) {
|
||||
fetch(factory, meta, true)
|
||||
}
|
||||
|
||||
logger.info { "Loading plugin ${plugin.name} into ${context.name}" }
|
||||
plugin.attach(context)
|
||||
plugins.add(plugin)
|
||||
return plugin
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a plugin using its factory
|
||||
*/
|
||||
@Deprecated("Use immutable contexts instead")
|
||||
internal fun <T : Plugin> load(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T =
|
||||
load(factory(meta, context))
|
||||
|
||||
/**
|
||||
* Remove a plugin from [PluginManager]
|
||||
*/
|
||||
@Deprecated("Use immutable contexts instead")
|
||||
public fun remove(plugin: Plugin) {
|
||||
if (plugins.contains(plugin)) {
|
||||
Global.logger.info { "Removing plugin ${plugin.name} from ${context.name}" }
|
||||
plugin.detach()
|
||||
plugins.remove(plugin)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an existing plugin with given meta or load new one using provided factory
|
||||
*/
|
||||
@Deprecated("Use immutable contexts instead")
|
||||
public fun <T : Plugin> fetch(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY, recursive: Boolean = true): T {
|
||||
val loaded = get(factory.type, factory.tag, recursive)
|
||||
return when {
|
||||
loaded == null -> load(factory(meta, context))
|
||||
loaded.meta == meta -> loaded // if meta is the same, return existing plugin
|
||||
else -> throw RuntimeException("Can't load plugin with tag ${factory.tag}. Plugin with this tag and different configuration already exists in context.")
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated("Use immutable contexts instead")
|
||||
public fun <T : Plugin> fetch(
|
||||
factory: PluginFactory<T>,
|
||||
recursive: Boolean = true,
|
||||
metaBuilder: MetaBuilder.() -> Unit,
|
||||
): T = fetch(factory, Meta(metaBuilder), recursive)
|
||||
getByType(T::class, factory.tag, recursive)
|
||||
|
||||
override fun iterator(): Iterator<Plugin> = plugins.iterator()
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a plugin with given meta from the context. If the plugin (with given meta) is already registered, it is returned.
|
||||
* Otherwise, new child context with the plugin is created. In the later case the context could be retrieved from the plugin.
|
||||
*/
|
||||
public inline fun <reified T : Plugin> Context.fetch(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T {
|
||||
public inline fun <reified T : Plugin> Context.request(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T {
|
||||
val existing = plugins[factory]
|
||||
return if (existing != null && existing.meta == meta) existing
|
||||
else {
|
||||
buildContext {
|
||||
buildContext(name = this@request.name + factory.tag.name) {
|
||||
plugin(factory, meta)
|
||||
}.plugins[factory]!!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated("Replace with request", ReplaceWith("request(factory, meta)"))
|
||||
public inline fun <reified T : Plugin> Context.fetch(factory: PluginFactory<T>, meta: Meta = Meta.EMPTY): T =
|
||||
request(factory, meta)
|
@ -1,5 +1,6 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
|
||||
@ -9,6 +10,7 @@ import space.kscience.dataforge.meta.MetaRepr
|
||||
*
|
||||
* @author Alexander Nozik
|
||||
*/
|
||||
@Serializable
|
||||
public data class PluginTag(
|
||||
val name: String,
|
||||
val group: String = "",
|
||||
|
@ -1,35 +1,35 @@
|
||||
package space.kscience.dataforge.properties
|
||||
|
||||
import space.kscience.dataforge.meta.Config
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.set
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.ObservableMutableMeta
|
||||
import space.kscience.dataforge.meta.transformations.MetaConverter
|
||||
import space.kscience.dataforge.meta.transformations.nullableItemToObject
|
||||
import space.kscience.dataforge.meta.transformations.nullableObjectToMetaItem
|
||||
import space.kscience.dataforge.meta.transformations.nullableMetaToObject
|
||||
import space.kscience.dataforge.meta.transformations.nullableObjectToMeta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
|
||||
@DFExperimental
|
||||
public class ConfigProperty<T : Any>(
|
||||
public val config: Config,
|
||||
public class MetaProperty<T : Any>(
|
||||
public val meta: ObservableMutableMeta,
|
||||
public val name: Name,
|
||||
public val converter: MetaConverter<T>,
|
||||
) : Property<T?> {
|
||||
|
||||
override var value: T?
|
||||
get() = converter.nullableItemToObject(config[name])
|
||||
get() = converter.nullableMetaToObject(meta[name])
|
||||
set(value) {
|
||||
config[name] = converter.nullableObjectToMetaItem(value)
|
||||
meta[name] = converter.nullableObjectToMeta(value) ?: Meta.EMPTY
|
||||
}
|
||||
|
||||
override fun onChange(owner: Any?, callback: (T?) -> Unit) {
|
||||
config.onChange(owner) { name, oldItem, newItem ->
|
||||
if (name.startsWith(this.name) && oldItem != newItem) callback(converter.nullableItemToObject(newItem))
|
||||
meta.onChange(owner) { name ->
|
||||
if (name.startsWith(this@MetaProperty.name)) callback(converter.nullableMetaToObject(this[name]))
|
||||
}
|
||||
}
|
||||
|
||||
override fun removeChangeListener(owner: Any?) {
|
||||
config.removeListener(owner)
|
||||
meta.removeListener(owner)
|
||||
}
|
||||
}
|
@ -1,13 +1,14 @@
|
||||
package space.kscience.dataforge.properties
|
||||
|
||||
import space.kscience.dataforge.meta.ItemPropertyProvider
|
||||
|
||||
import space.kscience.dataforge.meta.Scheme
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.reflect.KMutableProperty1
|
||||
|
||||
@DFExperimental
|
||||
public fun <P : ItemPropertyProvider, T : Any> P.property(property: KMutableProperty1<P, T?>): Property<T?> =
|
||||
public fun <S : Scheme, T : Any> S.property(property: KMutableProperty1<S, T?>): Property<T?> =
|
||||
object : Property<T?> {
|
||||
override var value: T?
|
||||
get() = property.get(this@property)
|
||||
@ -16,15 +17,15 @@ public fun <P : ItemPropertyProvider, T : Any> P.property(property: KMutableProp
|
||||
}
|
||||
|
||||
override fun onChange(owner: Any?, callback: (T?) -> Unit) {
|
||||
this@property.onChange(this) { name, oldItem, newItem ->
|
||||
if (name.startsWith(property.name.toName()) && oldItem != newItem) {
|
||||
this@property.meta.onChange(this) { name ->
|
||||
if (name.startsWith(property.name.parseAsName(true))) {
|
||||
callback(property.get(this@property))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun removeChangeListener(owner: Any?) {
|
||||
this@property.removeListener(this@property)
|
||||
this@property.meta.removeListener(this@property)
|
||||
}
|
||||
|
||||
}
|
@ -16,7 +16,7 @@
|
||||
package space.kscience.dataforge.provider
|
||||
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.jvm.JvmInline
|
||||
|
||||
/**
|
||||
@ -33,11 +33,7 @@ public value class Path(public val tokens: List<PathToken>) : Iterable<PathToken
|
||||
public companion object {
|
||||
public const val PATH_SEGMENT_SEPARATOR: String = "/"
|
||||
|
||||
public fun parse(path: String): Path {
|
||||
val head = path.substringBefore(PATH_SEGMENT_SEPARATOR)
|
||||
val tail = path.substringAfter(PATH_SEGMENT_SEPARATOR)
|
||||
return PathToken.parse(head).asPath() + parse(tail)
|
||||
}
|
||||
public fun parse(path: String): Path = Path(path.split(PATH_SEGMENT_SEPARATOR).map { PathToken.parse(it) })
|
||||
}
|
||||
}
|
||||
|
||||
@ -65,9 +61,10 @@ public data class PathToken(val name: Name, val target: String? = null) {
|
||||
|
||||
public companion object {
|
||||
public const val TARGET_SEPARATOR: String = "::"
|
||||
public fun parse(token: String): PathToken {
|
||||
|
||||
public fun parse(token: String, cache: Boolean = false): PathToken {
|
||||
val target = token.substringBefore(TARGET_SEPARATOR, "")
|
||||
val name = token.substringAfter(TARGET_SEPARATOR).toName()
|
||||
val name = token.substringAfter(TARGET_SEPARATOR).parseAsName(cache)
|
||||
if (target.contains("[")) TODO("target separators in queries are not supported")
|
||||
return PathToken(name, target)
|
||||
}
|
||||
|
@ -75,10 +75,8 @@ public inline fun <reified T : Any> Provider.provide(path: String, targetOverrid
|
||||
/**
|
||||
* Typed top level content
|
||||
*/
|
||||
public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> {
|
||||
return content(target).mapValues {
|
||||
type.safeCast(it.value) ?: error("The type of element $it is ${it::class} but $type is expected")
|
||||
}
|
||||
public fun <T : Any> Provider.top(target: String, type: KClass<out T>): Map<Name, T> = content(target).mapValues {
|
||||
type.safeCast(it.value) ?: error("The type of element ${it.value} is ${it.value::class} but $type is expected")
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1,8 +1,7 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.appendLeft
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.appendFirst
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -12,24 +11,22 @@ class ContextTest {
|
||||
override val tag get() = PluginTag("test")
|
||||
|
||||
override fun content(target: String): Map<Name, Any> {
|
||||
return when(target){
|
||||
"test" -> listOf("a", "b", "c.d").associate { it.toName() to it.toName() }
|
||||
return when (target) {
|
||||
"test" -> listOf("a", "b", "c.d").associate { Name.parse(it) to Name.parse(it) }
|
||||
else -> emptyMap()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testPluginManager() {
|
||||
val context = Global.buildContext{
|
||||
name("test")
|
||||
val context = Context("test") {
|
||||
plugin(DummyPlugin())
|
||||
}
|
||||
val members = context.gather<Name>("test")
|
||||
assertEquals(3, members.count())
|
||||
members.forEach {
|
||||
assertEquals(it.key, it.value.appendLeft("test"))
|
||||
assertEquals(it.key, it.value.appendFirst("test"))
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -14,7 +14,7 @@ internal class TestScheme : Scheme() {
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
class ItemPropertiesTest {
|
||||
class MetaPropertiesTest {
|
||||
@Test
|
||||
fun testBinding() {
|
||||
val scheme = TestScheme.empty()
|
@ -0,0 +1,14 @@
|
||||
package space.kscience.dataforge.provider
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
class PathTest {
|
||||
@Test
|
||||
fun testParse(){
|
||||
val nameString = "a.b.c.d"
|
||||
val pathString = "a.b/c.d"
|
||||
assertEquals(1, Path.parse(nameString).length)
|
||||
assertEquals(2, Path.parse(pathString).length)
|
||||
}
|
||||
}
|
@ -2,13 +2,14 @@ package space.kscience.dataforge.context
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
public class ConsoleLogManager : AbstractPlugin(), LogManager {
|
||||
|
||||
override fun logger(name: Name): Logger = Logger { tag, body ->
|
||||
val message: String = body.safe
|
||||
when (tag) {
|
||||
// TODO depends on https://youtrack.jetbrains.com/issue/KT-33595/
|
||||
LogManager.DEBUG -> console.asDynamic().debug("[${context.name}] $name: $message")
|
||||
LogManager.INFO -> console.info("[${context.name}] $name: $message")
|
||||
LogManager.WARNING -> console.warn("[${context.name}] $name: $message")
|
||||
LogManager.ERROR -> console.error("[${context.name}] $name: $message")
|
||||
@ -22,11 +23,10 @@ public class ConsoleLogManager : AbstractPlugin(), LogManager {
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
public companion object : PluginFactory<ConsoleLogManager> {
|
||||
override fun invoke(meta: Meta, context: Context): ConsoleLogManager = ConsoleLogManager()
|
||||
override fun build(context: Context, meta: Meta): ConsoleLogManager = ConsoleLogManager()
|
||||
|
||||
override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.jsConsole")
|
||||
override val type: KClass<out ConsoleLogManager> = ConsoleLogManager::class
|
||||
}
|
||||
}
|
||||
|
||||
internal actual val globalLoggerFactory: PluginFactory<out LogManager> = ConsoleLogManager
|
||||
internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = ConsoleLogManager
|
||||
|
@ -3,7 +3,6 @@ package space.kscience.dataforge.context
|
||||
import org.slf4j.LoggerFactory
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
public class SlfLogManager : AbstractPlugin(), LogManager {
|
||||
|
||||
@ -24,11 +23,10 @@ public class SlfLogManager : AbstractPlugin(), LogManager {
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
public companion object : PluginFactory<SlfLogManager> {
|
||||
override fun invoke(meta: Meta, context: Context): SlfLogManager = SlfLogManager()
|
||||
override fun build(context: Context, meta: Meta): SlfLogManager = SlfLogManager()
|
||||
|
||||
override val tag: PluginTag = PluginTag(group = PluginTag.DATAFORGE_GROUP, name = "log.kotlinLogging")
|
||||
override val type: KClass<out SlfLogManager> = SlfLogManager::class
|
||||
}
|
||||
}
|
||||
|
||||
internal actual val globalLoggerFactory: PluginFactory<out LogManager> = SlfLogManager
|
||||
internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = SlfLogManager
|
||||
|
@ -16,35 +16,32 @@
|
||||
|
||||
package space.kscience.dataforge.descriptors
|
||||
|
||||
import space.kscience.dataforge.values.ValueType
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
@MustBeDocumented
|
||||
annotation class Attribute(
|
||||
val key: String,
|
||||
val value: String
|
||||
)
|
||||
|
||||
@MustBeDocumented
|
||||
annotation class Attributes(
|
||||
val attrs: Array<Attribute>
|
||||
)
|
||||
|
||||
@MustBeDocumented
|
||||
annotation class ItemDef(
|
||||
val info: String = "",
|
||||
val multiple: Boolean = false,
|
||||
val required: Boolean = false
|
||||
)
|
||||
|
||||
@Target(AnnotationTarget.PROPERTY)
|
||||
@MustBeDocumented
|
||||
annotation class ValueDef(
|
||||
val type: Array<ValueType> = [ValueType.STRING],
|
||||
val def: String = "",
|
||||
val allowed: Array<String> = [],
|
||||
val enumeration: KClass<*> = Any::class
|
||||
)
|
||||
//@MustBeDocumented
|
||||
//annotation class Attribute(
|
||||
// val key: String,
|
||||
// val value: String
|
||||
//)
|
||||
//
|
||||
//@MustBeDocumented
|
||||
//annotation class Attributes(
|
||||
// val attrs: Array<Attribute>
|
||||
//)
|
||||
//
|
||||
//@MustBeDocumented
|
||||
//annotation class ItemDef(
|
||||
// val info: String = "",
|
||||
// val multiple: Boolean = false,
|
||||
// val required: Boolean = false
|
||||
//)
|
||||
//
|
||||
//@Target(AnnotationTarget.PROPERTY)
|
||||
//@MustBeDocumented
|
||||
//annotation class ValueDef(
|
||||
// val type: Array<ValueType> = [ValueType.STRING],
|
||||
// val def: String = "",
|
||||
// val allowed: Array<String> = [],
|
||||
// val enumeration: KClass<*> = Any::class
|
||||
//)
|
||||
|
||||
///**
|
||||
// * Description text for meta property, node or whole object
|
||||
|
@ -1,30 +1,32 @@
|
||||
package space.kscience.dataforge.provider
|
||||
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.PluginBuilder
|
||||
import space.kscience.dataforge.context.gather
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.reflect.KClass
|
||||
import kotlin.reflect.full.findAnnotation
|
||||
|
||||
|
||||
@DFExperimental
|
||||
public val KClass<*>.dfType: String
|
||||
get() = findAnnotation<Type>()?.id ?: simpleName ?: ""
|
||||
public val KClass<*>.dfId: String
|
||||
get() = findAnnotation<DfId>()?.id ?: simpleName ?: ""
|
||||
|
||||
/**
|
||||
* Provide an object with given name inferring target from its type using [Type] annotation
|
||||
* Provide an object with given name inferring target from its type using [DfId] annotation
|
||||
*/
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> Provider.provideByType(name: String): T? {
|
||||
val target = T::class.dfType
|
||||
val target = T::class.dfId
|
||||
return provide(target, name)
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
|
||||
val target = T::class.dfType
|
||||
val target = T::class.dfId
|
||||
return top(target)
|
||||
}
|
||||
|
||||
@ -33,5 +35,15 @@ public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
|
||||
*/
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> Context.gather(inherit: Boolean = true): Map<Name, T> =
|
||||
gather<T>(T::class.dfType, inherit)
|
||||
gather<T>(T::class.dfId, inherit)
|
||||
|
||||
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> PluginBuilder.provides(items: Map<Name, T>) {
|
||||
provides(T::class.dfId, items)
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> PluginBuilder.provides(vararg items: Named) {
|
||||
provides(T::class.dfId, *items)
|
||||
}
|
||||
|
@ -1,4 +1,4 @@
|
||||
package space.kscience.dataforge.context
|
||||
|
||||
|
||||
internal actual val globalLoggerFactory: PluginFactory<out LogManager> = DefaultLogManager
|
||||
internal actual fun getGlobalLoggerFactory(): PluginFactory<out LogManager> = DefaultLogManager
|
||||
|
23
dataforge-data/README.md
Normal file
23
dataforge-data/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-data
|
||||
|
||||
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-data:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-data:0.7.0")
|
||||
}
|
||||
```
|
@ -1,23 +1,18 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
kscience{
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
useCoroutines()
|
||||
}
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
commonMain{
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
api(kotlin("reflect"))
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
api(project(":dataforge-meta"))
|
||||
api(kotlin("reflect"))
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.EXPERIMENTAL
|
||||
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
|
||||
}
|
||||
|
@ -0,0 +1,65 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
import kotlin.reflect.KType
|
||||
|
||||
/**
|
||||
* Remove all values with keys starting with [name]
|
||||
*/
|
||||
internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
|
||||
val toRemove = keys.filter { it.startsWith(name) }
|
||||
toRemove.forEach(::remove)
|
||||
}
|
||||
|
||||
/**
|
||||
* An action that caches results on-demand and recalculates them on source push
|
||||
*/
|
||||
public abstract class AbstractAction<in T : Any, R : Any>(
|
||||
public val outputType: KType,
|
||||
) : Action<T, R> {
|
||||
|
||||
/**
|
||||
* Generate initial content of the output
|
||||
*/
|
||||
protected abstract fun DataSetBuilder<R>.generate(
|
||||
data: DataSet<T>,
|
||||
meta: Meta,
|
||||
)
|
||||
|
||||
/**
|
||||
* Update part of the data set when given [updateKey] is triggered by the source
|
||||
*/
|
||||
protected open fun DataSourceBuilder<R>.update(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
updateKey: Name,
|
||||
) {
|
||||
// By default, recalculate the whole dataset
|
||||
generate(dataSet, meta)
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
override fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
): DataSet<R> = if (dataSet is DataSource) {
|
||||
DataSource(outputType, dataSet){
|
||||
generate(dataSet, meta)
|
||||
|
||||
launch {
|
||||
dataSet.updates.collect { name ->
|
||||
update(dataSet, meta, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DataTree<R>(outputType) {
|
||||
generate(dataSet, meta)
|
||||
}
|
||||
}
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import space.kscience.dataforge.data.DataSet
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
@ -9,13 +8,12 @@ import space.kscience.dataforge.misc.DFExperimental
|
||||
* A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute].
|
||||
*/
|
||||
public interface Action<in T : Any, out R : Any> {
|
||||
|
||||
/**
|
||||
* Transform the data in the node, producing a new node. By default it is assumed that all calculations are lazy
|
||||
* Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy
|
||||
* so not actual computation is started at this moment.
|
||||
*
|
||||
* [scope] context used to compute the initial result, also it is used for updates propagation
|
||||
*/
|
||||
public suspend fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY, scope: CoroutineScope? = null): DataSet<R>
|
||||
public fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY): DataSet<R>
|
||||
|
||||
public companion object
|
||||
}
|
||||
@ -26,16 +24,17 @@ public interface Action<in T : Any, out R : Any> {
|
||||
public infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> {
|
||||
// TODO introduce composite action and add optimize by adding action to the list
|
||||
return object : Action<T, R> {
|
||||
override suspend fun execute(dataSet: DataSet<T>, meta: Meta, scope: CoroutineScope?): DataSet<R> {
|
||||
return action.execute(this@then.execute(dataSet, meta, scope), meta, scope)
|
||||
}
|
||||
|
||||
override fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
): DataSet<R> = action.execute(this@then.execute(dataSet, meta), meta)
|
||||
}
|
||||
}
|
||||
|
||||
@DFExperimental
|
||||
public suspend fun <T : Any, R : Any> DataSet<T>.transformWith(
|
||||
action: Action<T, R>,
|
||||
public operator fun <T : Any, R : Any> Action<T, R>.invoke(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
scope: CoroutineScope? = null,
|
||||
): DataSet<R> = action.execute(this, meta, scope)
|
||||
): DataSet<R> = execute(dataSet, meta)
|
||||
|
||||
|
@ -1,12 +1,8 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import kotlinx.coroutines.flow.map
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFBuilder
|
||||
@ -29,65 +25,71 @@ public data class ActionEnv(
|
||||
* Action environment
|
||||
*/
|
||||
@DFBuilder
|
||||
public class MapActionBuilder<T, R>(public var name: Name, public var meta: MetaBuilder, public val actionMeta: Meta) {
|
||||
public class MapActionBuilder<T, R>(
|
||||
public var name: Name,
|
||||
public var meta: MutableMeta,
|
||||
public val actionMeta: Meta,
|
||||
@PublishedApi internal var outputType: KType,
|
||||
) {
|
||||
|
||||
public lateinit var result: suspend ActionEnv.(T) -> R
|
||||
|
||||
/**
|
||||
* Set unsafe [outputType] for the resulting data. Be sure that it is correct.
|
||||
*/
|
||||
public fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(T) -> R1) {
|
||||
this.outputType = outputType
|
||||
result = f;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the result of goal
|
||||
*/
|
||||
public fun result(f: suspend ActionEnv.(T) -> R) {
|
||||
public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(T) -> R1) {
|
||||
outputType = typeOf<R1>()
|
||||
result = f;
|
||||
}
|
||||
}
|
||||
|
||||
@PublishedApi
|
||||
internal class MapAction<in T : Any, out R : Any>(
|
||||
private val outputType: KType,
|
||||
internal class MapAction<in T : Any, R : Any>(
|
||||
outputType: KType,
|
||||
private val block: MapActionBuilder<T, R>.() -> Unit,
|
||||
) : Action<T, R> {
|
||||
) : AbstractAction<T, R>(outputType) {
|
||||
|
||||
override suspend fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
scope: CoroutineScope?,
|
||||
): DataSet<R> {
|
||||
suspend fun mapOne(data: NamedData<T>): NamedData<R> {
|
||||
// Creating a new environment for action using **old** name, old meta and task meta
|
||||
val env = ActionEnv(data.name, data.meta, meta)
|
||||
private fun DataSetBuilder<R>.mapOne(name: Name, data: Data<T>, meta: Meta) {
|
||||
// Creating a new environment for action using **old** name, old meta and task meta
|
||||
val env = ActionEnv(name, data.meta, meta)
|
||||
|
||||
//applying transformation from builder
|
||||
val builder = MapActionBuilder<T, R>(
|
||||
data.name,
|
||||
data.meta.toMutableMeta(), // using data meta
|
||||
meta
|
||||
).apply(block)
|
||||
//applying transformation from builder
|
||||
val builder = MapActionBuilder<T, R>(
|
||||
name,
|
||||
data.meta.toMutableMeta(), // using data meta
|
||||
meta,
|
||||
outputType
|
||||
).apply(block)
|
||||
|
||||
//getting new name
|
||||
val newName = builder.name
|
||||
//getting new name
|
||||
val newName = builder.name
|
||||
|
||||
//getting new meta
|
||||
val newMeta = builder.meta.seal()
|
||||
//getting new meta
|
||||
val newMeta = builder.meta.seal()
|
||||
|
||||
@OptIn(DFInternal::class) val newData = Data(outputType, newMeta, dependencies = listOf(data)) {
|
||||
builder.result(env, data.await())
|
||||
}
|
||||
//setting the data node
|
||||
return newData.named(newName)
|
||||
@OptIn(DFInternal::class)
|
||||
val newData = Data(builder.outputType, newMeta, dependencies = listOf(data)) {
|
||||
builder.result(env, data.await())
|
||||
}
|
||||
//setting the data node
|
||||
data(newName, newData)
|
||||
}
|
||||
|
||||
val flow = dataSet.flow().map(::mapOne)
|
||||
override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
|
||||
data.forEach { mapOne(it.name, it.data, meta) }
|
||||
}
|
||||
|
||||
return ActiveDataTree(outputType) {
|
||||
populate(flow)
|
||||
scope?.launch {
|
||||
dataSet.updates.collect { name ->
|
||||
//clear old nodes
|
||||
remove(name)
|
||||
//collect new items
|
||||
populate(dataSet.flowChildren(name).map(::mapOne))
|
||||
}
|
||||
}
|
||||
}
|
||||
override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
|
||||
remove(updateKey)
|
||||
dataSet[updateKey]?.let { mapOne(updateKey, it, meta) }
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1,28 +1,34 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.flow
|
||||
import kotlinx.coroutines.flow.fold
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.misc.DFBuilder
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
|
||||
public class JoinGroup<T : Any, R : Any>(public var name: String, internal val set: DataSet<T>) {
|
||||
public class JoinGroup<T : Any, R : Any>(
|
||||
public var name: String,
|
||||
internal val set: DataSet<T>,
|
||||
@PublishedApi internal var outputType: KType,
|
||||
) {
|
||||
|
||||
public var meta: MetaBuilder = MetaBuilder()
|
||||
public var meta: MutableMeta = MutableMeta()
|
||||
|
||||
public lateinit var result: suspend ActionEnv.(Map<Name, T>) -> R
|
||||
public lateinit var result: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R
|
||||
|
||||
public fun result(f: suspend ActionEnv.(Map<Name, T>) -> R) {
|
||||
internal fun <R1 : R> result(outputType: KType, f: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R1) {
|
||||
this.outputType = outputType
|
||||
this.result = f;
|
||||
}
|
||||
|
||||
public inline fun <reified R1 : R> result(noinline f: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R1) {
|
||||
outputType = typeOf<R1>()
|
||||
this.result = f;
|
||||
}
|
||||
|
||||
@ -30,31 +36,30 @@ public class JoinGroup<T : Any, R : Any>(public var name: String, internal val s
|
||||
|
||||
@DFBuilder
|
||||
public class ReduceGroupBuilder<T : Any, R : Any>(
|
||||
private val inputType: KType,
|
||||
private val scope: CoroutineScope,
|
||||
public val actionMeta: Meta,
|
||||
private val outputType: KType,
|
||||
) {
|
||||
private val groupRules: MutableList<suspend (DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
|
||||
private val groupRules: MutableList<(DataSet<T>) -> List<JoinGroup<T, R>>> = ArrayList();
|
||||
|
||||
/**
|
||||
* introduce grouping by meta value
|
||||
*/
|
||||
public fun byValue(tag: String, defaultTag: String = "@default", action: JoinGroup<T, R>.() -> Unit) {
|
||||
groupRules += { node ->
|
||||
GroupRule.byMetaValue(scope, tag, defaultTag).gather(node).map {
|
||||
JoinGroup<T, R>(it.key, it.value).apply(action)
|
||||
GroupRule.byMetaValue(tag, defaultTag).gather(node).map {
|
||||
JoinGroup<T, R>(it.key, it.value, outputType).apply(action)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public fun group(
|
||||
groupName: String,
|
||||
filter: suspend (Name, Data<T>) -> Boolean,
|
||||
predicate: (Name, Meta) -> Boolean,
|
||||
action: JoinGroup<T, R>.() -> Unit,
|
||||
) {
|
||||
groupRules += { source ->
|
||||
listOf(
|
||||
JoinGroup<T, R>(groupName, source.filter(filter)).apply(action)
|
||||
JoinGroup<T, R>(groupName, source.filter(predicate), outputType).apply(action)
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -62,30 +67,27 @@ public class ReduceGroupBuilder<T : Any, R : Any>(
|
||||
/**
|
||||
* Apply transformation to the whole node
|
||||
*/
|
||||
public fun result(resultName: String, f: suspend ActionEnv.(Map<Name, T>) -> R) {
|
||||
public fun result(resultName: String, f: suspend ActionEnv.(Map<Name, ValueWithMeta<T>>) -> R) {
|
||||
groupRules += { node ->
|
||||
listOf(JoinGroup<T, R>(resultName, node).apply { result(f) })
|
||||
listOf(JoinGroup<T, R>(resultName, node, outputType).apply { result(outputType, f) })
|
||||
}
|
||||
}
|
||||
|
||||
internal suspend fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> {
|
||||
return groupRules.flatMap { it.invoke(input) }
|
||||
}
|
||||
internal fun buildGroups(input: DataSet<T>): List<JoinGroup<T, R>> =
|
||||
groupRules.flatMap { it.invoke(input) }
|
||||
|
||||
}
|
||||
|
||||
@PublishedApi
|
||||
internal class ReduceAction<T : Any, R : Any>(
|
||||
private val inputType: KType,
|
||||
outputType: KType,
|
||||
private val action: ReduceGroupBuilder<T, R>.() -> Unit,
|
||||
) : CachingAction<T, R>(outputType) {
|
||||
//TODO optimize reduction. Currently the whole action recalculates on push
|
||||
) : AbstractAction<T, R>(outputType) {
|
||||
//TODO optimize reduction. Currently, the whole action recalculates on push
|
||||
|
||||
|
||||
override fun CoroutineScope.transform(set: DataSet<T>, meta: Meta, key: Name): Flow<NamedData<R>> = flow {
|
||||
ReduceGroupBuilder<T, R>(inputType, this@transform, meta).apply(action).buildGroups(set).forEach { group ->
|
||||
val dataFlow: Map<Name, Data<T>> = group.set.flow().fold(HashMap()) { acc, value ->
|
||||
override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
|
||||
ReduceGroupBuilder<T, R>(meta, outputType).apply(action).buildGroups(data).forEach { group ->
|
||||
val dataFlow: Map<Name, Data<T>> = group.set.asSequence().fold(HashMap()) { acc, value ->
|
||||
acc.apply {
|
||||
acc[value.name] = value.data
|
||||
}
|
||||
@ -95,13 +97,13 @@ internal class ReduceAction<T : Any, R : Any>(
|
||||
|
||||
val groupMeta = group.meta
|
||||
|
||||
val env = ActionEnv(groupName.toName(), groupMeta, meta)
|
||||
val env = ActionEnv(groupName.parseAsName(), groupMeta, meta)
|
||||
@OptIn(DFInternal::class) val res: Data<R> = dataFlow.reduceToData(
|
||||
outputType,
|
||||
group.outputType,
|
||||
meta = groupMeta
|
||||
) { group.result.invoke(env, it) }
|
||||
|
||||
emit(res.named(env.name))
|
||||
data(env.name, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -110,7 +112,6 @@ internal class ReduceAction<T : Any, R : Any>(
|
||||
* A one-to-one mapping action
|
||||
*/
|
||||
@DFExperimental
|
||||
@Suppress("FunctionName")
|
||||
public inline fun <reified T : Any, reified R : Any> Action.Companion.reduce(
|
||||
noinline builder: ReduceGroupBuilder<T, R>.() -> Unit,
|
||||
): Action<T, R> = ReduceAction(typeOf<T>(), typeOf<R>(), builder)
|
||||
): Action<T, R> = ReduceAction(typeOf<R>(), builder)
|
||||
|
@ -1,18 +1,13 @@
|
||||
package space.kscience.dataforge.actions
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.FlowPreview
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.data.*
|
||||
import space.kscience.dataforge.meta.Laminate
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.collections.set
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
@ -20,10 +15,15 @@ import kotlin.reflect.typeOf
|
||||
|
||||
public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val meta: Meta) {
|
||||
|
||||
public class FragmentRule<T : Any, R : Any>(public val name: Name, public var meta: MetaBuilder) {
|
||||
public class FragmentRule<T : Any, R : Any>(
|
||||
public val name: Name,
|
||||
public var meta: MutableMeta,
|
||||
@PublishedApi internal var outputType: KType,
|
||||
) {
|
||||
public lateinit var result: suspend (T) -> R
|
||||
|
||||
public fun result(f: suspend (T) -> R) {
|
||||
public inline fun <reified R1 : R> result(noinline f: suspend (T) -> R1) {
|
||||
this.outputType = typeOf<R1>()
|
||||
result = f;
|
||||
}
|
||||
}
|
||||
@ -36,7 +36,7 @@ public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val me
|
||||
* @param rule the rule to transform fragment name and meta using
|
||||
*/
|
||||
public fun fragment(name: String, rule: FragmentRule<T, R>.() -> Unit) {
|
||||
fragments[name.toName()] = rule
|
||||
fragments[name.parseAsName()] = rule
|
||||
}
|
||||
}
|
||||
|
||||
@ -45,52 +45,48 @@ public class SplitBuilder<T : Any, R : Any>(public val name: Name, public val me
|
||||
*/
|
||||
@PublishedApi
|
||||
internal class SplitAction<T : Any, R : Any>(
|
||||
private val outputType: KType,
|
||||
outputType: KType,
|
||||
private val action: SplitBuilder<T, R>.() -> Unit,
|
||||
) : Action<T, R> {
|
||||
) : AbstractAction<T, R>(outputType) {
|
||||
|
||||
@OptIn(FlowPreview::class)
|
||||
override suspend fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
scope: CoroutineScope?,
|
||||
): DataSet<R> {
|
||||
private fun DataSetBuilder<R>.splitOne(name: Name, data: Data<T>, meta: Meta) {
|
||||
val laminate = Laminate(data.meta, meta)
|
||||
|
||||
suspend fun splitOne(data: NamedData<T>): Flow<NamedData<R>> {
|
||||
val laminate = Laminate(data.meta, meta)
|
||||
|
||||
val split = SplitBuilder<T, R>(data.name, data.meta).apply(action)
|
||||
val split = SplitBuilder<T, R>(name, data.meta).apply(action)
|
||||
|
||||
|
||||
// apply individual fragment rules to result
|
||||
return split.fragments.entries.asFlow().map { (fragmentName, rule) ->
|
||||
val env = SplitBuilder.FragmentRule<T, R>(fragmentName, laminate.toMutableMeta()).apply(rule)
|
||||
//data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
|
||||
@OptIn(DFInternal::class) Data(outputType, meta = env.meta, dependencies = listOf(data)) {
|
||||
// apply individual fragment rules to result
|
||||
split.fragments.forEach { (fragmentName, rule) ->
|
||||
val env = SplitBuilder.FragmentRule<T, R>(
|
||||
fragmentName,
|
||||
laminate.toMutableMeta(),
|
||||
outputType
|
||||
).apply(rule)
|
||||
//data.map<R>(outputType, meta = env.meta) { env.result(it) }.named(fragmentName)
|
||||
|
||||
data(
|
||||
fragmentName,
|
||||
@Suppress("OPT_IN_USAGE") Data(outputType, meta = env.meta, dependencies = listOf(data)) {
|
||||
env.result(data.await())
|
||||
}.named(fragmentName)
|
||||
}
|
||||
}
|
||||
|
||||
return ActiveDataTree<R>(outputType) {
|
||||
populate(dataSet.flow().flatMapConcat(transform = ::splitOne))
|
||||
scope?.launch {
|
||||
dataSet.updates.collect { name ->
|
||||
//clear old nodes
|
||||
remove(name)
|
||||
//collect new items
|
||||
populate(dataSet.flowChildren(name).flatMapConcat(transform = ::splitOne))
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override fun DataSetBuilder<R>.generate(data: DataSet<T>, meta: Meta) {
|
||||
data.forEach { splitOne(it.name, it.data, meta) }
|
||||
}
|
||||
|
||||
override fun DataSourceBuilder<R>.update(dataSet: DataSet<T>, meta: Meta, updateKey: Name) {
|
||||
remove(updateKey)
|
||||
dataSet[updateKey]?.let { splitOne(updateKey, it, meta) }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Action that splits each incoming element into a number of fragments defined in builder
|
||||
*/
|
||||
@DFExperimental
|
||||
@Suppress("FunctionName")
|
||||
public inline fun <T : Any, reified R : Any> Action.Companion.split(
|
||||
noinline builder: SplitBuilder<T, R>.() -> Unit,
|
||||
): Action<T, R> = SplitAction(typeOf<R>(), builder)
|
@ -1,118 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
/**
|
||||
* A mutable [DataTree.Companion.active]. It
|
||||
*/
|
||||
public class ActiveDataTree<T : Any>(
|
||||
override val dataType: KType,
|
||||
) : DataTree<T>, DataSetBuilder<T>, ActiveDataSet<T> {
|
||||
private val mutex = Mutex()
|
||||
private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
|
||||
|
||||
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = mutex.withLock {
|
||||
treeItems.filter { !it.key.body.startsWith("@") }
|
||||
}
|
||||
|
||||
private val _updates = MutableSharedFlow<Name>()
|
||||
|
||||
override val updates: Flow<Name>
|
||||
get() = _updates
|
||||
|
||||
private suspend fun remove(token: NameToken) {
|
||||
mutex.withLock {
|
||||
if (treeItems.remove(token) != null) {
|
||||
_updates.emit(token.asName())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun remove(name: Name) {
|
||||
if (name.isEmpty()) error("Can't remove the root node")
|
||||
(getItem(name.cutLast()).tree as? ActiveDataTree)?.remove(name.lastOrNull()!!)
|
||||
}
|
||||
|
||||
private suspend fun set(token: NameToken, data: Data<T>) {
|
||||
mutex.withLock {
|
||||
treeItems[token] = DataTreeItem.Leaf(data)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun getOrCreateNode(token: NameToken): ActiveDataTree<T> =
|
||||
(treeItems[token] as? DataTreeItem.Node<T>)?.tree as? ActiveDataTree<T>
|
||||
?: ActiveDataTree<T>(dataType).also {
|
||||
mutex.withLock {
|
||||
treeItems[token] = DataTreeItem.Node(it)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun getOrCreateNode(name: Name): ActiveDataTree<T> {
|
||||
return when (name.length) {
|
||||
0 -> this
|
||||
1 -> getOrCreateNode(name.firstOrNull()!!)
|
||||
else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, data: Data<T>?) {
|
||||
if (data == null) {
|
||||
remove(name)
|
||||
} else {
|
||||
when (name.length) {
|
||||
0 -> error("Can't add data with empty name")
|
||||
1 -> set(name.firstOrNull()!!, data)
|
||||
2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
|
||||
}
|
||||
}
|
||||
_updates.emit(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy given data set and mirror its changes to this [ActiveDataTree] in [this@setAndObserve]. Returns an update [Job]
|
||||
*/
|
||||
public fun CoroutineScope.setAndObserve(name: Name, dataSet: DataSet<T>): Job = launch {
|
||||
emit(name, dataSet)
|
||||
dataSet.updates.collect { nameInBranch ->
|
||||
emit(name + nameInBranch, dataSet.getData(nameInBranch))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a dynamic tree. Initial data is placed synchronously. Updates are propagated via [updatesScope]
|
||||
*/
|
||||
@Suppress("FunctionName")
|
||||
public suspend fun <T : Any> ActiveDataTree(
|
||||
type: KType,
|
||||
block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): ActiveDataTree<T> {
|
||||
val tree = ActiveDataTree<T>(type)
|
||||
tree.block()
|
||||
return tree
|
||||
}
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend inline fun <reified T : Any> ActiveDataTree(
|
||||
crossinline block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): ActiveDataTree<T> = ActiveDataTree<T>(typeOf<T>()).apply { block() }
|
||||
|
||||
|
||||
public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
|
||||
name: Name,
|
||||
noinline block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): Unit = emit(name, ActiveDataTree(typeOf<T>(), block))
|
||||
|
||||
public suspend inline fun <reified T : Any> ActiveDataTree<T>.emit(
|
||||
name: String,
|
||||
noinline block: suspend ActiveDataTree<T>.() -> Unit,
|
||||
): Unit = emit(name.toName(), ActiveDataTree(typeOf<T>(), block))
|
@ -1,52 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.actions.Action
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.startsWith
|
||||
import kotlin.reflect.KType
|
||||
|
||||
/**
|
||||
* Remove all values with keys starting with [name]
|
||||
*/
|
||||
internal fun MutableMap<Name, *>.removeWhatStartsWith(name: Name) {
|
||||
val toRemove = keys.filter { it.startsWith(name) }
|
||||
toRemove.forEach(::remove)
|
||||
}
|
||||
|
||||
/**
|
||||
* An action that caches results on-demand and recalculates them on source push
|
||||
*/
|
||||
public abstract class CachingAction<in T : Any, out R : Any>(
|
||||
public val outputType: KType,
|
||||
) : Action<T, R> {
|
||||
|
||||
protected abstract fun CoroutineScope.transform(
|
||||
set: DataSet<T>,
|
||||
meta: Meta,
|
||||
key: Name = Name.EMPTY,
|
||||
): Flow<NamedData<R>>
|
||||
|
||||
override suspend fun execute(
|
||||
dataSet: DataSet<T>,
|
||||
meta: Meta,
|
||||
scope: CoroutineScope?,
|
||||
): DataSet<R> = ActiveDataTree<R>(outputType) {
|
||||
coroutineScope {
|
||||
populate(transform(dataSet, meta))
|
||||
}
|
||||
scope?.let {
|
||||
dataSet.updates.collect {
|
||||
//clear old nodes
|
||||
remove(it)
|
||||
//collect new items
|
||||
populate(scope.transform(dataSet, meta, it))
|
||||
//FIXME if the target is data, updates are fired twice
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -5,7 +5,7 @@ import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.meta.isEmpty
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
@ -14,8 +14,8 @@ import kotlin.reflect.typeOf
|
||||
/**
|
||||
* A data element characterized by its meta
|
||||
*/
|
||||
@Type(Data.TYPE)
|
||||
public interface Data<out T : Any> : Goal<T>, MetaRepr {
|
||||
@DfId(Data.TYPE)
|
||||
public interface Data<out T> : Goal<T>, MetaRepr {
|
||||
/**
|
||||
* Type marker for the data. The type is known before the calculation takes place so it could be checked.
|
||||
*/
|
||||
@ -49,6 +49,7 @@ public interface Data<out T : Any> : Goal<T>, MetaRepr {
|
||||
/**
|
||||
* An empty data containing only meta
|
||||
*/
|
||||
@OptIn(DelicateCoroutinesApi::class)
|
||||
public fun empty(meta: Meta): Data<Nothing> = object : Data<Nothing> {
|
||||
override val type: KType = TYPE_OF_NOTHING
|
||||
override val meta: Meta = meta
|
||||
@ -72,7 +73,7 @@ private class LazyData<T : Any>(
|
||||
override val type: KType,
|
||||
override val meta: Meta = Meta.EMPTY,
|
||||
additionalContext: CoroutineContext = EmptyCoroutineContext,
|
||||
dependencies: Collection<Data<*>> = emptyList(),
|
||||
dependencies: Collection<Goal<*>> = emptyList(),
|
||||
block: suspend () -> T,
|
||||
) : Data<T>, LazyGoal<T>(additionalContext, dependencies, block)
|
||||
|
||||
@ -82,13 +83,17 @@ public class StaticData<T : Any>(
|
||||
override val meta: Meta = Meta.EMPTY,
|
||||
) : Data<T>, StaticGoal<T>(value)
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public inline fun <reified T : Any> Data(value: T, meta: Meta = Meta.EMPTY): StaticData<T> =
|
||||
StaticData(typeOf<T>(), value, meta)
|
||||
|
||||
@Suppress("FunctionName")
|
||||
@DFInternal
|
||||
public fun <T : Any> Data(
|
||||
type: KType,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
context: CoroutineContext = EmptyCoroutineContext,
|
||||
dependencies: Collection<Data<*>> = emptyList(),
|
||||
dependencies: Collection<Goal<*>> = emptyList(),
|
||||
block: suspend () -> T,
|
||||
): Data<T> = LazyData(type, meta, context, dependencies, block)
|
||||
|
||||
@ -97,6 +102,6 @@ public fun <T : Any> Data(
|
||||
public inline fun <reified T : Any> Data(
|
||||
meta: Meta = Meta.EMPTY,
|
||||
context: CoroutineContext = EmptyCoroutineContext,
|
||||
dependencies: Collection<Data<*>> = emptyList(),
|
||||
dependencies: Collection<Goal<*>> = emptyList(),
|
||||
noinline block: suspend () -> T,
|
||||
): Data<T> = Data(typeOf<T>(), meta, context, dependencies, block)
|
||||
|
@ -1,11 +1,15 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.emptyFlow
|
||||
import kotlinx.coroutines.flow.mapNotNull
|
||||
import space.kscience.dataforge.data.Data.Companion.TYPE_OF_NOTHING
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.set
|
||||
import space.kscience.dataforge.names.*
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.endsWith
|
||||
import space.kscience.dataforge.names.parseAsName
|
||||
import kotlin.reflect.KType
|
||||
|
||||
public interface DataSet<out T : Any> {
|
||||
@ -16,23 +20,19 @@ public interface DataSet<out T : Any> {
|
||||
public val dataType: KType
|
||||
|
||||
/**
|
||||
* Traverse this provider or its child. The order is not guaranteed.
|
||||
* [root] points to a root name for traversal. If it is empty, traverse this source, if it points to a [Data],
|
||||
* return flow, that contains single [Data], if it points to a node with children, return children.
|
||||
* Meta-data associated with this node. If no meta is provided, returns [Meta.EMPTY].
|
||||
*/
|
||||
public fun flow(): Flow<NamedData<T>>
|
||||
public val meta: Meta
|
||||
|
||||
/**
|
||||
* Traverse this [DataSet] returning named data instances. The order is not guaranteed.
|
||||
*/
|
||||
public operator fun iterator(): Iterator<NamedData<T>>
|
||||
|
||||
/**
|
||||
* Get data with given name.
|
||||
*/
|
||||
public suspend fun getData(name: Name): Data<T>?
|
||||
|
||||
/**
|
||||
* Get a snapshot of names of top level children of given node. Empty if node does not exist or is a leaf.
|
||||
*/
|
||||
public suspend fun listTop(prefix: Name = Name.EMPTY): List<Name> =
|
||||
flow().map { it.name }.filter { it.startsWith(prefix) && (it.length == prefix.length + 1) }.toList()
|
||||
// By default traverses the whole tree. Could be optimized in descendants
|
||||
public operator fun get(name: Name): Data<T>?
|
||||
|
||||
public companion object {
|
||||
public val META_KEY: Name = "@meta".asName()
|
||||
@ -42,17 +42,35 @@ public interface DataSet<out T : Any> {
|
||||
*/
|
||||
public val EMPTY: DataSet<Nothing> = object : DataSet<Nothing> {
|
||||
override val dataType: KType = TYPE_OF_NOTHING
|
||||
override val meta: Meta get() = Meta.EMPTY
|
||||
|
||||
private val nothing: Nothing get() = error("this is nothing")
|
||||
override fun iterator(): Iterator<NamedData<Nothing>> = emptySequence<NamedData<Nothing>>().iterator()
|
||||
|
||||
override fun flow(): Flow<NamedData<Nothing>> = emptyFlow()
|
||||
|
||||
override suspend fun getData(name: Name): Data<Nothing>? = null
|
||||
override fun get(name: Name): Data<Nothing>? = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public interface ActiveDataSet<T : Any> : DataSet<T> {
|
||||
public fun <T : Any> DataSet<T>.asSequence(): Sequence<NamedData<T>> = object : Sequence<NamedData<T>> {
|
||||
override fun iterator(): Iterator<NamedData<T>> = this@asSequence.iterator()
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a single [Data] in this [DataSet]. Throw error if it is not single.
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.single(): NamedData<T> = asSequence().single()
|
||||
|
||||
public fun <T : Any> DataSet<T>.asIterable(): Iterable<NamedData<T>> = object : Iterable<NamedData<T>> {
|
||||
override fun iterator(): Iterator<NamedData<T>> = this@asIterable.iterator()
|
||||
}
|
||||
|
||||
public operator fun <T : Any> DataSet<T>.get(name: String): Data<T>? = get(name.parseAsName())
|
||||
|
||||
/**
|
||||
* A [DataSet] with propagated updates.
|
||||
*/
|
||||
public interface DataSource<out T : Any> : DataSet<T>, CoroutineScope {
|
||||
|
||||
/**
|
||||
* A flow of updated item names. Updates are propagated in a form of [Flow] of names of updated nodes.
|
||||
* Those can include new data items and replacement of existing ones. The replaced items could update existing data content
|
||||
@ -60,30 +78,38 @@ public interface ActiveDataSet<T : Any> : DataSet<T> {
|
||||
*
|
||||
*/
|
||||
public val updates: Flow<Name>
|
||||
|
||||
/**
|
||||
* Stop generating updates from this [DataSource]
|
||||
*/
|
||||
public fun close() {
|
||||
coroutineContext[Job]?.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is ActiveDataSet) updates else emptyFlow()
|
||||
|
||||
/**
|
||||
* Flow all data nodes with names starting with [branchName]
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.flowChildren(branchName: Name): Flow<NamedData<T>> = this@flowChildren.flow().filter {
|
||||
it.name.startsWith(branchName)
|
||||
}
|
||||
public val <T : Any> DataSet<T>.updates: Flow<Name> get() = if (this is DataSource) updates else emptyFlow()
|
||||
//
|
||||
///**
|
||||
// * Flow all data nodes with names starting with [branchName]
|
||||
// */
|
||||
//public fun <T : Any> DataSet<T>.children(branchName: Name): Sequence<NamedData<T>> =
|
||||
// this@children.asSequence().filter {
|
||||
// it.name.startsWith(branchName)
|
||||
// }
|
||||
|
||||
/**
|
||||
* Start computation for all goals in data node and return a job for the whole node
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.startAll(coroutineScope: CoroutineScope): Job = coroutineScope.launch {
|
||||
flow().map {
|
||||
asIterable().map {
|
||||
it.launch(this@launch)
|
||||
}.toList().joinAll()
|
||||
}.joinAll()
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSet<T>.join(): Unit = coroutineScope { startAll(this).join() }
|
||||
public suspend fun <T : Any> DataSet<T>.computeAndJoinAll(): Unit = coroutineScope { startAll(this).join() }
|
||||
|
||||
public suspend fun DataSet<*>.toMeta(): Meta = Meta {
|
||||
flow().collect {
|
||||
public fun DataSet<*>.toMeta(): Meta = Meta {
|
||||
forEach {
|
||||
if (it.name.endsWith(DataSet.META_KEY)) {
|
||||
set(it.name, it.meta)
|
||||
} else {
|
||||
@ -95,4 +121,4 @@ public suspend fun DataSet<*>.toMeta(): Meta = Meta {
|
||||
}
|
||||
}
|
||||
|
||||
public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { getData(it)?.named(it) }
|
||||
public val <T : Any> DataSet<T>.updatesWithData: Flow<NamedData<T>> get() = updates.mapNotNull { get(it)?.named(it) }
|
@ -1,14 +1,11 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.isEmpty
|
||||
import space.kscience.dataforge.names.plus
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.reflect.KType
|
||||
|
||||
public interface DataSetBuilder<in T : Any> {
|
||||
@ -17,129 +14,152 @@ public interface DataSetBuilder<in T : Any> {
|
||||
/**
|
||||
* Remove all data items starting with [name]
|
||||
*/
|
||||
public suspend fun remove(name: Name)
|
||||
public fun remove(name: Name)
|
||||
|
||||
public suspend fun emit(name: Name, data: Data<T>?)
|
||||
public fun data(name: Name, data: Data<T>?)
|
||||
|
||||
/**
|
||||
* Set a current state of given [dataSet] into a branch [name]. Does not propagate updates
|
||||
*/
|
||||
public suspend fun emit(name: Name, dataSet: DataSet<T>) {
|
||||
public fun node(name: Name, dataSet: DataSet<T>) {
|
||||
//remove previous items
|
||||
if (name != Name.EMPTY) {
|
||||
remove(name)
|
||||
}
|
||||
|
||||
//Set new items
|
||||
dataSet.flow().collect {
|
||||
emit(name + it.name, it.data)
|
||||
dataSet.forEach {
|
||||
data(name + it.name, it.data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append data to node
|
||||
* Set meta for the given node
|
||||
*/
|
||||
public suspend infix fun String.put(data: Data<T>): Unit = emit(toName(), data)
|
||||
public fun meta(name: Name, meta: Meta)
|
||||
|
||||
/**
|
||||
* Append node
|
||||
*/
|
||||
public suspend infix fun String.put(dataSet: DataSet<T>): Unit = emit(toName(), dataSet)
|
||||
|
||||
/**
|
||||
* Build and append node
|
||||
*/
|
||||
public suspend infix fun String.put(block: suspend DataSetBuilder<T>.() -> Unit): Unit = emit(toName(), block)
|
||||
}
|
||||
|
||||
private class SubSetBuilder<in T : Any>(
|
||||
/**
|
||||
* Define meta in this [DataSet]
|
||||
*/
|
||||
public fun <T : Any> DataSetBuilder<T>.meta(value: Meta): Unit = meta(Name.EMPTY, value)
|
||||
|
||||
/**
|
||||
* Define meta in this [DataSet]
|
||||
*/
|
||||
public fun <T : Any> DataSetBuilder<T>.meta(mutableMeta: MutableMeta.() -> Unit): Unit = meta(Meta(mutableMeta))
|
||||
|
||||
@PublishedApi
|
||||
internal class SubSetBuilder<in T : Any>(
|
||||
private val parent: DataSetBuilder<T>,
|
||||
private val branch: Name,
|
||||
) : DataSetBuilder<T> {
|
||||
override val dataType: KType get() = parent.dataType
|
||||
|
||||
override suspend fun remove(name: Name) {
|
||||
override fun remove(name: Name) {
|
||||
parent.remove(branch + name)
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, data: Data<T>?) {
|
||||
parent.emit(branch + name, data)
|
||||
override fun data(name: Name, data: Data<T>?) {
|
||||
parent.data(branch + name, data)
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, dataSet: DataSet<T>) {
|
||||
parent.emit(branch + name, dataSet)
|
||||
override fun node(name: Name, dataSet: DataSet<T>) {
|
||||
parent.node(branch + name, dataSet)
|
||||
}
|
||||
|
||||
override fun meta(name: Name, meta: Meta) {
|
||||
parent.meta(branch + name, meta)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: Name, block: suspend DataSetBuilder<T>.() -> Unit) {
|
||||
SubSetBuilder(this, name).apply { block() }
|
||||
public inline fun <T : Any> DataSetBuilder<T>.node(
|
||||
name: Name,
|
||||
crossinline block: DataSetBuilder<T>.() -> Unit,
|
||||
) {
|
||||
if (name.isEmpty()) block() else SubSetBuilder(this, name).block()
|
||||
}
|
||||
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, data: Data<T>) {
|
||||
emit(name.toName(), data)
|
||||
public fun <T : Any> DataSetBuilder<T>.data(name: String, value: Data<T>) {
|
||||
data(Name.parse(name), value)
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, set: DataSet<T>) {
|
||||
this.emit(name.toName(), set)
|
||||
public fun <T : Any> DataSetBuilder<T>.node(name: String, set: DataSet<T>) {
|
||||
node(Name.parse(name), set)
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(name: String, block: suspend DataSetBuilder<T>.() -> Unit): Unit =
|
||||
this@emit.emit(name.toName(), block)
|
||||
public inline fun <T : Any> DataSetBuilder<T>.node(
|
||||
name: String,
|
||||
crossinline block: DataSetBuilder<T>.() -> Unit,
|
||||
): Unit = node(Name.parse(name), block)
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.emit(data: NamedData<T>) {
|
||||
emit(data.name, data.data)
|
||||
public fun <T : Any> DataSetBuilder<T>.set(value: NamedData<T>) {
|
||||
data(value.name, value.data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Produce lazy [Data] and emit it into the [DataSetBuilder]
|
||||
*/
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
name: String,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline producer: suspend () -> T,
|
||||
) {
|
||||
val data = Data(meta, block = producer)
|
||||
emit(name, data)
|
||||
data(name, data)
|
||||
}
|
||||
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.produce(
|
||||
name: Name,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline producer: suspend () -> T,
|
||||
) {
|
||||
val data = Data(meta, block = producer)
|
||||
emit(name, data)
|
||||
data(name, data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a static data with the fixed value
|
||||
*/
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(name: String, data: T, meta: Meta = Meta.EMPTY): Unit =
|
||||
emit(name, Data.static(data, meta))
|
||||
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(name: Name, data: T, meta: Meta = Meta.EMPTY): Unit =
|
||||
emit(name, Data.static(data, meta))
|
||||
|
||||
public suspend inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
name: String,
|
||||
data: T,
|
||||
metaBuilder: MetaBuilder.() -> Unit,
|
||||
): Unit = emit(name.toName(), Data.static(data, Meta(metaBuilder)))
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): Unit = data(name, Data.static(data, meta))
|
||||
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
name: Name,
|
||||
data: T,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
): Unit = data(name, Data.static(data, meta))
|
||||
|
||||
public inline fun <reified T : Any> DataSetBuilder<T>.static(
|
||||
name: String,
|
||||
data: T,
|
||||
mutableMeta: MutableMeta.() -> Unit,
|
||||
): Unit = data(Name.parse(name), Data.static(data, Meta(mutableMeta)))
|
||||
|
||||
/**
|
||||
* Update data with given node data and meta with node meta.
|
||||
*/
|
||||
@DFExperimental
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.populate(tree: DataSet<T>): Unit = coroutineScope {
|
||||
tree.flow().collect {
|
||||
public fun <T : Any> DataSetBuilder<T>.populateFrom(tree: DataSet<T>): Unit {
|
||||
tree.forEach {
|
||||
//TODO check if the place is occupied
|
||||
emit(it.name, it.data)
|
||||
data(it.name, it.data)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun <T : Any> DataSetBuilder<T>.populate(flow: Flow<NamedData<T>>) {
|
||||
flow.collect {
|
||||
emit(it.name, it.data)
|
||||
//public fun <T : Any> DataSetBuilder<T>.populateFrom(flow: Flow<NamedData<T>>) {
|
||||
// flow.collect {
|
||||
// data(it.name, it.data)
|
||||
// }
|
||||
//}
|
||||
|
||||
public fun <T : Any> DataSetBuilder<T>.populateFrom(sequence: Sequence<NamedData<T>>) {
|
||||
sequence.forEach {
|
||||
data(it.name, it.data)
|
||||
}
|
||||
}
|
||||
|
@ -1,18 +1,25 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.emitAll
|
||||
import kotlinx.coroutines.flow.flow
|
||||
import kotlinx.coroutines.flow.map
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public sealed class DataTreeItem<out T : Any> {
|
||||
public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>()
|
||||
public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>()
|
||||
|
||||
public abstract val meta: Meta
|
||||
|
||||
public class Node<out T : Any>(public val tree: DataTree<T>) : DataTreeItem<T>() {
|
||||
override val meta: Meta get() = tree.meta
|
||||
}
|
||||
|
||||
public class Leaf<out T : Any>(public val data: Data<T>) : DataTreeItem<T>() {
|
||||
override val meta: Meta get() = data.meta
|
||||
}
|
||||
}
|
||||
|
||||
public val <T : Any> DataTreeItem<T>.type: KType
|
||||
@ -24,63 +31,79 @@ public val <T : Any> DataTreeItem<T>.type: KType
|
||||
/**
|
||||
* A tree-like [DataSet] grouped into the node. All data inside the node must inherit its type
|
||||
*/
|
||||
@Type(DataTree.TYPE)
|
||||
@DfId(DataTree.TYPE)
|
||||
public interface DataTree<out T : Any> : DataSet<T> {
|
||||
|
||||
/**
|
||||
* Children items of this [DataTree] provided asynchronously
|
||||
* Top-level children items of this [DataTree]
|
||||
*/
|
||||
public suspend fun items(): Map<NameToken, DataTreeItem<T>>
|
||||
public val items: Map<NameToken, DataTreeItem<T>>
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> = flow {
|
||||
items().forEach { (token, childItem: DataTreeItem<T>) ->
|
||||
if(!token.body.startsWith("@")) {
|
||||
override val meta: Meta get() = items[META_ITEM_NAME_TOKEN]?.meta ?: Meta.EMPTY
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
items.forEach { (token, childItem: DataTreeItem<T>) ->
|
||||
if (!token.body.startsWith("@")) {
|
||||
when (childItem) {
|
||||
is DataTreeItem.Leaf -> emit(childItem.data.named(token.asName()))
|
||||
is DataTreeItem.Node -> emitAll(childItem.tree.flow().map { it.named(token + it.name) })
|
||||
is DataTreeItem.Leaf -> yield(childItem.data.named(token.asName()))
|
||||
is DataTreeItem.Node -> yieldAll(childItem.tree.asSequence().map { it.named(token + it.name) })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun listTop(prefix: Name): List<Name> =
|
||||
getItem(prefix).tree?.items()?.keys?.map { prefix + it } ?: emptyList()
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? = when (name.length) {
|
||||
override fun get(name: Name): Data<T>? = when (name.length) {
|
||||
0 -> null
|
||||
1 -> items()[name.firstOrNull()!!].data
|
||||
else -> items()[name.firstOrNull()!!].tree?.getData(name.cutFirst())
|
||||
1 -> items[name.firstOrNull()!!].data
|
||||
else -> items[name.firstOrNull()!!].tree?.get(name.cutFirst())
|
||||
}
|
||||
|
||||
public companion object {
|
||||
public const val TYPE: String = "dataTree"
|
||||
|
||||
/**
|
||||
* A name token used to designate tree node meta
|
||||
*/
|
||||
public val META_ITEM_NAME_TOKEN: NameToken = NameToken("@meta")
|
||||
|
||||
@DFInternal
|
||||
public fun <T : Any> emptyWithType(type: KType, meta: Meta = Meta.EMPTY): DataTree<T> = object : DataTree<T> {
|
||||
override val items: Map<NameToken, DataTreeItem<T>> get() = emptyMap()
|
||||
override val dataType: KType get() = type
|
||||
override val meta: Meta get() = meta
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline fun <reified T : Any> empty(meta: Meta = Meta.EMPTY): DataTree<T> =
|
||||
emptyWithType<T>(typeOf<T>(), meta)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend fun <T: Any> DataSet<T>.getData(name: String): Data<T>? = getData(name.toName())
|
||||
public fun <T : Any> DataTree<T>.listChildren(prefix: Name): List<Name> =
|
||||
getItem(prefix).tree?.items?.keys?.map { prefix + it } ?: emptyList()
|
||||
|
||||
/**
|
||||
* Get a [DataTreeItem] with given [name] or null if the item does not exist
|
||||
*/
|
||||
public tailrec suspend fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
|
||||
public tailrec fun <T : Any> DataTree<T>.getItem(name: Name): DataTreeItem<T>? = when (name.length) {
|
||||
0 -> DataTreeItem.Node(this)
|
||||
1 -> items()[name.firstOrNull()]
|
||||
else -> items()[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
|
||||
1 -> items[name.firstOrNull()]
|
||||
else -> items[name.firstOrNull()!!].tree?.getItem(name.cutFirst())
|
||||
}
|
||||
|
||||
public val <T : Any> DataTreeItem<T>?.tree: DataTree<T>? get() = (this as? DataTreeItem.Node<T>)?.tree
|
||||
public val <T : Any> DataTreeItem<T>?.data: Data<T>? get() = (this as? DataTreeItem.Leaf<T>)?.data
|
||||
|
||||
/**
|
||||
* Flow of all children including nodes
|
||||
* A [Sequence] of all children including nodes
|
||||
*/
|
||||
public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> = flow {
|
||||
items().forEach { (head, item) ->
|
||||
emit(head.asName() to item)
|
||||
public fun <T : Any> DataTree<T>.traverseItems(): Sequence<Pair<Name, DataTreeItem<T>>> = sequence {
|
||||
items.forEach { (head, item) ->
|
||||
yield(head.asName() to item)
|
||||
if (item is DataTreeItem.Node) {
|
||||
val subSequence = item.tree.itemFlow()
|
||||
val subSequence = item.tree.traverseItems()
|
||||
.map { (name, data) -> (head.asName() + name) to data }
|
||||
emitAll(subSequence)
|
||||
yieldAll(subSequence)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -89,8 +112,8 @@ public fun <T : Any> DataTree<T>.itemFlow(): Flow<Pair<Name, DataTreeItem<T>>> =
|
||||
* Get a branch of this [DataTree] with a given [branchName].
|
||||
* The difference from similar method for [DataSet] is that internal logic is more simple and the return value is a [DataTree]
|
||||
*/
|
||||
public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> = object : DataTree<T> {
|
||||
override val dataType: KType get() = this@branch.dataType
|
||||
@OptIn(DFInternal::class)
|
||||
public fun <T : Any> DataTree<T>.branch(branchName: Name): DataTree<T> =
|
||||
getItem(branchName)?.tree ?: DataTree.emptyWithType(dataType)
|
||||
|
||||
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = getItem(branchName).tree?.items() ?: emptyMap()
|
||||
}
|
||||
public fun <T : Any> DataTree<T>.branch(branchName: String): DataTree<T> = branch(branchName.parseAsName())
|
||||
|
@ -0,0 +1,127 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.misc.ThreadSafe
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.collections.set
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.coroutineContext
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public interface DataSourceBuilder<T : Any> : DataSetBuilder<T>, DataSource<T> {
|
||||
override val updates: MutableSharedFlow<Name>
|
||||
}
|
||||
|
||||
/**
|
||||
* A mutable [DataTree] that propagates updates
|
||||
*/
|
||||
public class DataTreeBuilder<T : Any>(
|
||||
override val dataType: KType,
|
||||
coroutineContext: CoroutineContext,
|
||||
) : DataTree<T>, DataSourceBuilder<T> {
|
||||
|
||||
override val coroutineContext: CoroutineContext =
|
||||
coroutineContext + Job(coroutineContext[Job]) + GoalExecutionRestriction()
|
||||
|
||||
private val treeItems = HashMap<NameToken, DataTreeItem<T>>()
|
||||
|
||||
override val items: Map<NameToken, DataTreeItem<T>>
|
||||
get() = treeItems.filter { !it.key.body.startsWith("@") }
|
||||
|
||||
override val updates: MutableSharedFlow<Name> = MutableSharedFlow<Name>()
|
||||
|
||||
@ThreadSafe
|
||||
private fun remove(token: NameToken) {
|
||||
if (treeItems.remove(token) != null) {
|
||||
launch {
|
||||
updates.emit(token.asName())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun remove(name: Name) {
|
||||
if (name.isEmpty()) error("Can't remove the root node")
|
||||
(getItem(name.cutLast()).tree as? DataTreeBuilder)?.remove(name.lastOrNull()!!)
|
||||
}
|
||||
|
||||
@ThreadSafe
|
||||
private fun set(token: NameToken, data: Data<T>) {
|
||||
treeItems[token] = DataTreeItem.Leaf(data)
|
||||
}
|
||||
|
||||
@ThreadSafe
|
||||
private fun set(token: NameToken, node: DataTree<T>) {
|
||||
treeItems[token] = DataTreeItem.Node(node)
|
||||
}
|
||||
|
||||
private fun getOrCreateNode(token: NameToken): DataTreeBuilder<T> =
|
||||
(treeItems[token] as? DataTreeItem.Node<T>)?.tree as? DataTreeBuilder<T>
|
||||
?: DataTreeBuilder<T>(dataType, coroutineContext).also { set(token, it) }
|
||||
|
||||
private fun getOrCreateNode(name: Name): DataTreeBuilder<T> = when (name.length) {
|
||||
0 -> this
|
||||
1 -> getOrCreateNode(name.firstOrNull()!!)
|
||||
else -> getOrCreateNode(name.firstOrNull()!!).getOrCreateNode(name.cutFirst())
|
||||
}
|
||||
|
||||
override fun data(name: Name, data: Data<T>?) {
|
||||
if (data == null) {
|
||||
remove(name)
|
||||
} else {
|
||||
when (name.length) {
|
||||
0 -> error("Can't add data with empty name")
|
||||
1 -> set(name.firstOrNull()!!, data)
|
||||
2 -> getOrCreateNode(name.cutLast()).set(name.lastOrNull()!!, data)
|
||||
}
|
||||
}
|
||||
launch {
|
||||
updates.emit(name)
|
||||
}
|
||||
}
|
||||
|
||||
override fun meta(name: Name, meta: Meta) {
|
||||
val item = getItem(name)
|
||||
if (item is DataTreeItem.Leaf) error("TODO: Can't change meta of existing leaf item.")
|
||||
data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a dynamic [DataSource]. Initial data is placed synchronously.
|
||||
*/
|
||||
@DFInternal
|
||||
@Suppress("FunctionName")
|
||||
public fun <T : Any> DataSource(
|
||||
type: KType,
|
||||
parent: CoroutineScope,
|
||||
block: DataSourceBuilder<T>.() -> Unit,
|
||||
): DataTreeBuilder<T> = DataTreeBuilder<T>(type, parent.coroutineContext).apply(block)
|
||||
|
||||
@Suppress("OPT_IN_USAGE", "FunctionName")
|
||||
public inline fun <reified T : Any> DataSource(
|
||||
parent: CoroutineScope,
|
||||
crossinline block: DataSourceBuilder<T>.() -> Unit,
|
||||
): DataTreeBuilder<T> = DataSource(typeOf<T>(), parent) { block() }
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend inline fun <reified T : Any> DataSource(
|
||||
crossinline block: DataSourceBuilder<T>.() -> Unit = {},
|
||||
): DataTreeBuilder<T> = DataTreeBuilder<T>(typeOf<T>(), coroutineContext).apply { block() }
|
||||
|
||||
public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
|
||||
name: Name,
|
||||
parent: CoroutineScope,
|
||||
noinline block: DataSourceBuilder<T>.() -> Unit,
|
||||
): Unit = node(name, DataSource(parent, block))
|
||||
|
||||
public inline fun <reified T : Any> DataSourceBuilder<T>.emit(
|
||||
name: String,
|
||||
parent: CoroutineScope,
|
||||
noinline block: DataSourceBuilder<T>.() -> Unit,
|
||||
): Unit = node(Name.parse(name), DataSource(parent, block))
|
@ -67,7 +67,7 @@ public open class LazyGoal<T>(
|
||||
* If [GoalExecutionRestriction] is present in the [coroutineScope] context, the call could produce a error a warning
|
||||
* depending on the settings.
|
||||
*/
|
||||
@DFExperimental
|
||||
@OptIn(DFExperimental::class)
|
||||
override fun async(coroutineScope: CoroutineScope): Deferred<T> {
|
||||
val log = coroutineScope.coroutineContext[GoalLogger]
|
||||
// Check if context restricts goal computation
|
||||
|
@ -3,11 +3,25 @@ package space.kscience.dataforge.data
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
|
||||
public enum class GoalExecutionRestrictionPolicy {
|
||||
/**
|
||||
* Allow eager execution
|
||||
*/
|
||||
NONE,
|
||||
|
||||
/**
|
||||
* Give warning on eager execution
|
||||
*/
|
||||
WARNING,
|
||||
|
||||
/**
|
||||
* Throw error on eager execution
|
||||
*/
|
||||
ERROR
|
||||
}
|
||||
|
||||
/**
|
||||
* A special coroutine context key that allows or disallows goal execution during configuration time (eager execution).
|
||||
*/
|
||||
public class GoalExecutionRestriction(
|
||||
public val policy: GoalExecutionRestrictionPolicy = GoalExecutionRestrictionPolicy.ERROR,
|
||||
) : CoroutineContext.Element {
|
||||
|
@ -2,6 +2,9 @@ package space.kscience.dataforge.data
|
||||
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
|
||||
/**
|
||||
* Coroutine context element that provides logging capabilities
|
||||
*/
|
||||
public interface GoalLogger : CoroutineContext.Element {
|
||||
override val key: CoroutineContext.Key<*> get() = GoalLogger
|
||||
|
||||
|
@ -15,14 +15,13 @@
|
||||
*/
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
|
||||
public interface GroupRule {
|
||||
public suspend fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
|
||||
public fun <T : Any> gather(set: DataSet<T>): Map<String, DataSet<T>>
|
||||
|
||||
public companion object {
|
||||
/**
|
||||
@ -33,30 +32,45 @@ public interface GroupRule {
|
||||
* @param defaultTagValue
|
||||
* @return
|
||||
*/
|
||||
@OptIn(DFInternal::class)
|
||||
public fun byMetaValue(
|
||||
scope: CoroutineScope,
|
||||
key: String,
|
||||
defaultTagValue: String,
|
||||
): GroupRule = object : GroupRule {
|
||||
|
||||
override suspend fun <T : Any> gather(
|
||||
override fun <T : Any> gather(
|
||||
set: DataSet<T>,
|
||||
): Map<String, DataSet<T>> {
|
||||
val map = HashMap<String, ActiveDataTree<T>>()
|
||||
val map = HashMap<String, DataSet<T>>()
|
||||
|
||||
set.flow().collect { data ->
|
||||
val tagValue = data.meta[key]?.string ?: defaultTagValue
|
||||
map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(data.name, data.data)
|
||||
}
|
||||
if (set is DataSource) {
|
||||
set.forEach { data ->
|
||||
val tagValue: String = data.meta[key]?.string ?: defaultTagValue
|
||||
(map.getOrPut(tagValue) { DataTreeBuilder(set.dataType, set.coroutineContext) } as DataTreeBuilder<T>)
|
||||
.data(data.name, data.data)
|
||||
|
||||
scope.launch {
|
||||
set.updates.collect { name ->
|
||||
val data = set.getData(name)
|
||||
val tagValue = data?.meta[key].string ?: defaultTagValue
|
||||
map.getOrPut(tagValue) { ActiveDataTree(set.dataType) }.emit(name, data)
|
||||
set.launch {
|
||||
set.updates.collect { name ->
|
||||
val dataUpdate = set[name]
|
||||
|
||||
val updateTagValue = dataUpdate?.meta?.get(key)?.string ?: defaultTagValue
|
||||
map.getOrPut(updateTagValue) {
|
||||
DataSource(set.dataType, this) {
|
||||
data(name, dataUpdate)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
set.forEach { data ->
|
||||
val tagValue: String = data.meta[key]?.string ?: defaultTagValue
|
||||
(map.getOrPut(tagValue) { StaticDataTree(set.dataType) } as StaticDataTree<T>)
|
||||
.data(data.name, data.data)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return map
|
||||
}
|
||||
}
|
||||
|
@ -9,6 +9,9 @@ public interface NamedData<out T : Any> : Named, Data<T> {
|
||||
public val data: Data<T>
|
||||
}
|
||||
|
||||
public operator fun NamedData<*>.component1(): Name = name
|
||||
public operator fun <T: Any> NamedData<T>.component2(): Data<T> = data
|
||||
|
||||
private class NamedDataImpl<out T : Any>(
|
||||
override val name: Name,
|
||||
override val data: Data<T>,
|
||||
|
@ -1,7 +1,6 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.reflect.KType
|
||||
@ -12,15 +11,16 @@ internal class StaticDataTree<T : Any>(
|
||||
override val dataType: KType,
|
||||
) : DataSetBuilder<T>, DataTree<T> {
|
||||
|
||||
private val items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
|
||||
private val _items: MutableMap<NameToken, DataTreeItem<T>> = HashMap()
|
||||
|
||||
override suspend fun items(): Map<NameToken, DataTreeItem<T>> = items.filter { !it.key.body.startsWith("@") }
|
||||
override val items: Map<NameToken, DataTreeItem<T>>
|
||||
get() = _items.filter { !it.key.body.startsWith("@") }
|
||||
|
||||
override suspend fun remove(name: Name) {
|
||||
override fun remove(name: Name) {
|
||||
when (name.length) {
|
||||
0 -> error("Can't remove root tree node")
|
||||
1 -> items.remove(name.firstOrNull()!!)
|
||||
else -> (items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
|
||||
1 -> _items.remove(name.firstOrNull()!!)
|
||||
else -> (_items[name.firstOrNull()!!].tree as? StaticDataTree<T>)?.remove(name.cutFirst())
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,51 +28,55 @@ internal class StaticDataTree<T : Any>(
|
||||
0 -> this
|
||||
1 -> {
|
||||
val itemName = name.firstOrNull()!!
|
||||
(items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
|
||||
items[itemName] = DataTreeItem.Node(it)
|
||||
(_items[itemName].tree as? StaticDataTree<T>) ?: StaticDataTree<T>(dataType).also {
|
||||
_items[itemName] = DataTreeItem.Node(it)
|
||||
}
|
||||
}
|
||||
else -> getOrCreateNode(name.cutLast()).getOrCreateNode(name.lastOrNull()!!.asName())
|
||||
}
|
||||
|
||||
private suspend fun set(name: Name, item: DataTreeItem<T>?) {
|
||||
private fun set(name: Name, item: DataTreeItem<T>?) {
|
||||
if (name.isEmpty()) error("Can't set top level tree node")
|
||||
if (item == null) {
|
||||
remove(name)
|
||||
} else {
|
||||
getOrCreateNode(name.cutLast()).items[name.lastOrNull()!!] = item
|
||||
getOrCreateNode(name.cutLast())._items[name.lastOrNull()!!] = item
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, data: Data<T>?) {
|
||||
override fun data(name: Name, data: Data<T>?) {
|
||||
set(name, data?.let { DataTreeItem.Leaf(it) })
|
||||
}
|
||||
|
||||
override suspend fun emit(name: Name, dataSet: DataSet<T>) {
|
||||
override fun node(name: Name, dataSet: DataSet<T>) {
|
||||
if (dataSet is StaticDataTree) {
|
||||
set(name, DataTreeItem.Node(dataSet))
|
||||
} else {
|
||||
coroutineScope {
|
||||
dataSet.flow().collect {
|
||||
emit(name + it.name, it.data)
|
||||
}
|
||||
dataSet.forEach {
|
||||
data(name + it.name, it.data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun meta(name: Name, meta: Meta) {
|
||||
val item = getItem(name)
|
||||
if (item is DataTreeItem.Leaf) TODO("Can't change meta of existing leaf item.")
|
||||
data(name + DataTree.META_ITEM_NAME_TOKEN, Data.empty(meta))
|
||||
}
|
||||
}
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend fun <T : Any> DataTree(
|
||||
public inline fun <T : Any> DataTree(
|
||||
dataType: KType,
|
||||
block: suspend DataSetBuilder<T>.() -> Unit,
|
||||
block: DataSetBuilder<T>.() -> Unit,
|
||||
): DataTree<T> = StaticDataTree<T>(dataType).apply { block() }
|
||||
|
||||
@Suppress("FunctionName")
|
||||
public suspend inline fun <reified T : Any> DataTree(
|
||||
noinline block: suspend DataSetBuilder<T>.() -> Unit,
|
||||
public inline fun <reified T : Any> DataTree(
|
||||
noinline block: DataSetBuilder<T>.() -> Unit,
|
||||
): DataTree<T> = DataTree(typeOf<T>(), block)
|
||||
|
||||
@OptIn(DFExperimental::class)
|
||||
public suspend fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType){
|
||||
populate(this@seal)
|
||||
public fun <T : Any> DataSet<T>.seal(): DataTree<T> = DataTree(dataType) {
|
||||
populateFrom(this@seal)
|
||||
}
|
@ -4,8 +4,11 @@ import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.filter
|
||||
import kotlinx.coroutines.flow.map
|
||||
import kotlinx.coroutines.flow.mapNotNull
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.*
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
|
||||
|
||||
@ -13,34 +16,58 @@ import kotlin.reflect.KType
|
||||
* A stateless filtered [DataSet]
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.filter(
|
||||
predicate: suspend (Name, Data<T>) -> Boolean,
|
||||
): ActiveDataSet<T> = object : ActiveDataSet<T> {
|
||||
predicate: (Name, Meta) -> Boolean,
|
||||
): DataSource<T> = object : DataSource<T> {
|
||||
|
||||
override val dataType: KType get() = this@filter.dataType
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> =
|
||||
this@filter.flow().filter { predicate(it.name, it.data) }
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@filter as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? = this@filter.getData(name)?.takeIf {
|
||||
predicate(name, it)
|
||||
|
||||
override val meta: Meta get() = this@filter.meta
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
for (d in this@filter) {
|
||||
if (predicate(d.name, d.meta)) {
|
||||
yield(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun get(name: Name): Data<T>? = this@filter.get(name)?.takeIf {
|
||||
predicate(name, it.meta)
|
||||
}
|
||||
|
||||
override val updates: Flow<Name> = this@filter.updates.filter flowFilter@{ name ->
|
||||
val theData = this@filter.getData(name) ?: return@flowFilter false
|
||||
predicate(name, theData)
|
||||
val theData = this@filter[name] ?: return@flowFilter false
|
||||
predicate(name, theData.meta)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a wrapper data set with a given name prefix appended to all names
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) this
|
||||
else object : ActiveDataSet<T> {
|
||||
public fun <T : Any> DataSet<T>.withNamePrefix(prefix: Name): DataSet<T> = if (prefix.isEmpty()) {
|
||||
this
|
||||
} else object : DataSource<T> {
|
||||
|
||||
override val dataType: KType get() = this@withNamePrefix.dataType
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> = this@withNamePrefix.flow().map { it.data.named(prefix + it.name) }
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@withNamePrefix as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? =
|
||||
name.removeHeadOrNull(name)?.let { this@withNamePrefix.getData(it) }
|
||||
override val meta: Meta get() = this@withNamePrefix.meta
|
||||
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
for (d in this@withNamePrefix) {
|
||||
yield(d.data.named(prefix + d.name))
|
||||
}
|
||||
}
|
||||
|
||||
override fun get(name: Name): Data<T>? =
|
||||
name.removeFirstOrNull(name)?.let { this@withNamePrefix.get(it) }
|
||||
|
||||
override val updates: Flow<Name> get() = this@withNamePrefix.updates.map { prefix + it }
|
||||
}
|
||||
@ -50,22 +77,29 @@ else object : ActiveDataSet<T> {
|
||||
*/
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: Name): DataSet<T> = if (branchName.isEmpty()) {
|
||||
this
|
||||
} else object : ActiveDataSet<T> {
|
||||
} else object : DataSource<T> {
|
||||
override val dataType: KType get() = this@branch.dataType
|
||||
|
||||
override fun flow(): Flow<NamedData<T>> = this@branch.flow().mapNotNull {
|
||||
it.name.removeHeadOrNull(branchName)?.let { name ->
|
||||
it.data.named(name)
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@branch as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override val meta: Meta get() = this@branch.meta
|
||||
|
||||
override fun iterator(): Iterator<NamedData<T>> = iterator {
|
||||
for (d in this@branch) {
|
||||
d.name.removeFirstOrNull(branchName)?.let { name ->
|
||||
yield(d.data.named(name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun getData(name: Name): Data<T>? = this@branch.getData(branchName + name)
|
||||
override fun get(name: Name): Data<T>? = this@branch.get(branchName + name)
|
||||
|
||||
override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeHeadOrNull(branchName) }
|
||||
override val updates: Flow<Name> get() = this@branch.updates.mapNotNull { it.removeFirstOrNull(branchName) }
|
||||
}
|
||||
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.toName())
|
||||
public fun <T : Any> DataSet<T>.branch(branchName: String): DataSet<T> = this@branch.branch(branchName.parseAsName())
|
||||
|
||||
@DFExperimental
|
||||
public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = getData(Name.EMPTY)
|
||||
public suspend fun <T : Any> DataSet<T>.rootData(): Data<T>? = get(Name.EMPTY)
|
||||
|
||||
|
@ -1,20 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
|
||||
|
||||
/**
|
||||
* Get a metadata node for this set if it is present
|
||||
*/
|
||||
public suspend fun DataSet<*>.getMeta(): Meta? = getData(DataSet.META_KEY)?.meta
|
||||
|
||||
/**
|
||||
* Add meta-data node to a [DataSet]
|
||||
*/
|
||||
public suspend fun DataSetBuilder<*>.meta(meta: Meta): Unit = emit(DataSet.META_KEY, Data.empty(meta))
|
||||
|
||||
/**
|
||||
* Add meta-data node to a [DataSet]
|
||||
*/
|
||||
public suspend fun DataSetBuilder<*>.meta(metaBuilder: MetaBuilder.() -> Unit): Unit = meta(Meta(metaBuilder))
|
@ -1,18 +1,26 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaBuilder
|
||||
import space.kscience.dataforge.meta.MutableMeta
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.meta.toMutableMeta
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import kotlin.contracts.InvocationKind
|
||||
import kotlin.contracts.contract
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public data class ValueWithMeta<T>(val meta: Meta, val value: T)
|
||||
|
||||
public suspend fun <T : Any> Data<T>.awaitWithMeta(): ValueWithMeta<T> = ValueWithMeta(meta, await())
|
||||
|
||||
public data class NamedValueWithMeta<T>(val name: Name, val meta: Meta, val value: T)
|
||||
|
||||
public suspend fun <T : Any> NamedData<T>.awaitWithMeta(): NamedValueWithMeta<T> =
|
||||
NamedValueWithMeta(name, meta, await())
|
||||
|
||||
|
||||
/**
|
||||
* Lazily transform this data to another data. By convention [block] should not use external data (be pure).
|
||||
* @param coroutineContext additional [CoroutineContext] elements used for data computation.
|
||||
@ -28,7 +36,7 @@ public inline fun <T : Any, reified R : Any> Data<T>.map(
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine this data with the other data using [block]. See [map] for other details
|
||||
* Combine this data with the other data using [block]. See [Data::map] for other details
|
||||
*/
|
||||
public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
|
||||
other: Data<T2>,
|
||||
@ -48,13 +56,13 @@ public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
|
||||
public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline block: suspend (Collection<T>) -> R,
|
||||
crossinline block: suspend (List<ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
meta,
|
||||
coroutineContext,
|
||||
this
|
||||
) {
|
||||
block(map { it.await() })
|
||||
block(map { it.awaitWithMeta() })
|
||||
}
|
||||
|
||||
@DFInternal
|
||||
@ -62,17 +70,16 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
block: suspend (Map<K, T>) -> R,
|
||||
block: suspend (Map<K, ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
outputType,
|
||||
meta,
|
||||
coroutineContext,
|
||||
this.values
|
||||
) {
|
||||
block(mapValues { it.value.await() })
|
||||
block(mapValues { it.value.awaitWithMeta() })
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Lazily reduce a [Map] of [Data] with any static key.
|
||||
* @param K type of the map key
|
||||
@ -82,56 +89,91 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
|
||||
public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline block: suspend (Map<K, T>) -> R,
|
||||
crossinline block: suspend (Map<K, ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
meta,
|
||||
coroutineContext,
|
||||
this.values
|
||||
) {
|
||||
block(mapValues { it.value.await() })
|
||||
block(mapValues { it.value.awaitWithMeta() })
|
||||
}
|
||||
|
||||
//flow operations
|
||||
//Iterable operations
|
||||
|
||||
/**
|
||||
* Transform a [Flow] of [NamedData] to a single [Data].
|
||||
*/
|
||||
@DFInternal
|
||||
public suspend fun <T : Any, R : Any> Flow<NamedData<T>>.reduceToData(
|
||||
public inline fun <T : Any, R : Any> Iterable<Data<T>>.reduceToData(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
transformation: suspend (Flow<NamedData<T>>) -> R,
|
||||
crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
outputType,
|
||||
meta,
|
||||
coroutineContext,
|
||||
toList()
|
||||
) {
|
||||
transformation(this)
|
||||
transformation(map { it.awaitWithMeta() })
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.reduceToData(
|
||||
public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline transformation: suspend (Flow<NamedData<T>>) -> R,
|
||||
crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
|
||||
): Data<R> = reduceToData(typeOf<R>(), coroutineContext, meta) {
|
||||
transformation(it)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a flow of named data into a single [Data]
|
||||
*/
|
||||
public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.foldToData(
|
||||
public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.foldToData(
|
||||
initial: R,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline block: suspend (result: R, data: NamedData<T>) -> R,
|
||||
crossinline block: suspend (result: R, data: ValueWithMeta<T>) -> R,
|
||||
): Data<R> = reduceToData(
|
||||
coroutineContext, meta
|
||||
) {
|
||||
it.fold(initial, block)
|
||||
it.fold(initial) { acc, t -> block(acc, t) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform an [Iterable] of [NamedData] to a single [Data].
|
||||
*/
|
||||
@DFInternal
|
||||
public inline fun <T : Any, R : Any> Iterable<NamedData<T>>.reduceNamedToData(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
|
||||
): Data<R> = Data(
|
||||
outputType,
|
||||
meta,
|
||||
coroutineContext,
|
||||
toList()
|
||||
) {
|
||||
transformation(map { it.awaitWithMeta() })
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.reduceNamedToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
|
||||
): Data<R> = reduceNamedToData(typeOf<R>(), coroutineContext, meta) {
|
||||
transformation(it)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a [Iterable] of named data into a single [Data]
|
||||
*/
|
||||
public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.foldNamedToData(
|
||||
initial: R,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
|
||||
): Data<R> = reduceNamedToData(
|
||||
coroutineContext, meta
|
||||
) {
|
||||
it.fold(initial) { acc, t -> block(acc, t) }
|
||||
}
|
||||
|
||||
//DataSet operations
|
||||
@ -140,42 +182,40 @@ public suspend inline fun <T : Any, reified R : Any> Flow<NamedData<T>>.foldToDa
|
||||
public suspend fun <T : Any, R : Any> DataSet<T>.map(
|
||||
outputType: KType,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
metaTransform: MetaBuilder.() -> Unit = {},
|
||||
block: suspend (T) -> R,
|
||||
metaTransform: MutableMeta.() -> Unit = {},
|
||||
block: suspend (NamedValueWithMeta<T>) -> R,
|
||||
): DataTree<R> = DataTree<R>(outputType) {
|
||||
populate(
|
||||
flow().map {
|
||||
val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
|
||||
Data(outputType, newMeta, coroutineContext, listOf(it)) {
|
||||
block(it.await())
|
||||
}.named(it.name)
|
||||
forEach {
|
||||
val newMeta = it.meta.toMutableMeta().apply(metaTransform).seal()
|
||||
val d = Data(outputType, newMeta, coroutineContext, listOf(it)) {
|
||||
block(it.awaitWithMeta())
|
||||
}
|
||||
)
|
||||
data(it.name, d)
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
noinline metaTransform: MetaBuilder.() -> Unit = {},
|
||||
noinline block: suspend (T) -> R,
|
||||
noinline metaTransform: MutableMeta.() -> Unit = {},
|
||||
noinline block: suspend (NamedValueWithMeta<T>) -> R,
|
||||
): DataTree<R> = map(typeOf<R>(), coroutineContext, metaTransform, block)
|
||||
|
||||
public suspend fun <T : Any> DataSet<T>.forEach(block: suspend (NamedData<T>) -> Unit) {
|
||||
contract { callsInPlace(block, InvocationKind.EXACTLY_ONCE) }
|
||||
flow().collect {
|
||||
block(it)
|
||||
public inline fun <T : Any> DataSet<T>.forEach(block: (NamedData<T>) -> Unit) {
|
||||
for (d in this) {
|
||||
block(d)
|
||||
}
|
||||
}
|
||||
|
||||
public suspend inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
|
||||
public inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline transformation: suspend (Flow<NamedData<T>>) -> R,
|
||||
): Data<R> = flow().reduceToData(coroutineContext, meta, transformation)
|
||||
crossinline transformation: suspend (Iterable<NamedValueWithMeta<T>>) -> R,
|
||||
): Data<R> = asIterable().reduceNamedToData(coroutineContext, meta, transformation)
|
||||
|
||||
public suspend inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
|
||||
public inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
|
||||
initial: R,
|
||||
coroutineContext: CoroutineContext = EmptyCoroutineContext,
|
||||
meta: Meta = Meta.EMPTY,
|
||||
noinline block: suspend (result: R, data: NamedData<T>) -> R,
|
||||
): Data<R> = flow().foldToData(initial, coroutineContext, meta, block)
|
||||
crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
|
||||
): Data<R> = asIterable().foldNamedToData(initial, coroutineContext, meta, block)
|
@ -0,0 +1,2 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
@ -0,0 +1,85 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.filter
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.coroutines.CoroutineContext
|
||||
import kotlin.coroutines.EmptyCoroutineContext
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.full.isSubtypeOf
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
|
||||
/**
|
||||
* Cast the node to given type if the cast is possible or return null
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
|
||||
if (!this.type.isSubtypeOf(type)) {
|
||||
null
|
||||
} else {
|
||||
object : Data<R> by (this as Data<R>) {
|
||||
override val type: KType = type
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Select all data matching given type and filters. Does not modify paths
|
||||
*
|
||||
* @param predicate addition filtering condition based on item name and meta. By default, accepts all
|
||||
*/
|
||||
@OptIn(DFExperimental::class)
|
||||
public fun <R : Any> DataSet<*>.filterByType(
|
||||
type: KType,
|
||||
predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
|
||||
): DataSource<R> = object : DataSource<R> {
|
||||
override val dataType = type
|
||||
|
||||
override val coroutineContext: CoroutineContext
|
||||
get() = (this@filterByType as? DataSource)?.coroutineContext ?: EmptyCoroutineContext
|
||||
|
||||
override val meta: Meta get() = this@filterByType.meta
|
||||
|
||||
private fun checkDatum(name: Name, datum: Data<*>): Boolean = datum.type.isSubtypeOf(type)
|
||||
&& predicate(name, datum.meta)
|
||||
|
||||
override fun iterator(): Iterator<NamedData<R>> = iterator {
|
||||
for(d in this@filterByType){
|
||||
if(checkDatum(d.name,d.data)){
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
yield(d as NamedData<R>)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun get(name: Name): Data<R>? = this@filterByType[name]?.let { datum ->
|
||||
if (checkDatum(name, datum)) datum.castOrNull(type) else null
|
||||
}
|
||||
|
||||
override val updates: Flow<Name> = this@filterByType.updates.filter { name ->
|
||||
get(name)?.let { datum ->
|
||||
checkDatum(name, datum)
|
||||
} ?: false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Select a single datum of the appropriate type
|
||||
*/
|
||||
public inline fun <reified R : Any> DataSet<*>.filterByType(
|
||||
noinline predicate: (name: Name, meta: Meta) -> Boolean = { _, _ -> true },
|
||||
): DataSet<R> = filterByType(typeOf<R>(), predicate)
|
||||
|
||||
/**
|
||||
* Select a single datum if it is present and of given [type]
|
||||
*/
|
||||
public fun <R : Any> DataSet<*>.getByType(type: KType, name: Name): NamedData<R>? =
|
||||
get(name)?.castOrNull<R>(type)?.named(name)
|
||||
|
||||
public inline fun <reified R : Any> DataSet<*>.getByType(name: Name): NamedData<R>? =
|
||||
this@getByType.getByType(typeOf<R>(), name)
|
||||
|
||||
public inline fun <reified R : Any> DataSet<*>.getByType(name: String): NamedData<R>? =
|
||||
this@getByType.getByType(typeOf<R>(), Name.parse(name))
|
@ -0,0 +1,40 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.launch
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
|
||||
|
||||
/**
|
||||
* Append data to node
|
||||
*/
|
||||
context(DataSetBuilder<T>) public infix fun <T : Any> String.put(data: Data<T>): Unit =
|
||||
data(Name.parse(this), data)
|
||||
|
||||
/**
|
||||
* Append node
|
||||
*/
|
||||
context(DataSetBuilder<T>) public infix fun <T : Any> String.put(dataSet: DataSet<T>): Unit =
|
||||
node(Name.parse(this), dataSet)
|
||||
|
||||
/**
|
||||
* Build and append node
|
||||
*/
|
||||
context(DataSetBuilder<T>) public infix fun <T : Any> String.put(
|
||||
block: DataSetBuilder<T>.() -> Unit,
|
||||
): Unit = node(Name.parse(this), block)
|
||||
|
||||
/**
|
||||
* Copy given data set and mirror its changes to this [DataTreeBuilder] in [this@setAndObserve]. Returns an update [Job]
|
||||
*/
|
||||
context(DataSetBuilder<T>) public fun <T : Any> CoroutineScope.setAndWatch(
|
||||
name: Name,
|
||||
dataSet: DataSet<T>,
|
||||
): Job = launch {
|
||||
node(name, dataSet)
|
||||
dataSet.updates.collect { nameInBranch ->
|
||||
data(name + nameInBranch, dataSet.get(nameInBranch))
|
||||
}
|
||||
}
|
@ -1,64 +0,0 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.filter
|
||||
import kotlinx.coroutines.flow.map
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.matches
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.full.isSubtypeOf
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
|
||||
/**
|
||||
* Cast the node to given type if the cast is possible or return null
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
private fun <R : Any> Data<*>.castOrNull(type: KType): Data<R>? =
|
||||
if (!this.type.isSubtypeOf(type)) null else object : Data<R> by (this as Data<R>) {
|
||||
override val type: KType = type
|
||||
}
|
||||
|
||||
/**
|
||||
* Select all data matching given type and filters. Does not modify paths
|
||||
*/
|
||||
@OptIn(DFExperimental::class)
|
||||
@PublishedApi
|
||||
internal fun <R : Any> DataSet<*>.select(
|
||||
type: KType,
|
||||
namePattern: Name? = null,
|
||||
): ActiveDataSet<R> = object : ActiveDataSet<R> {
|
||||
override val dataType = type
|
||||
|
||||
|
||||
override fun flow(): Flow<NamedData<R>> = this@select.flow().filter { datum ->
|
||||
datum.type.isSubtypeOf(type) && (namePattern == null || datum.name.matches(namePattern))
|
||||
}.map {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
it as NamedData<R>
|
||||
}
|
||||
|
||||
override suspend fun getData(name: Name): Data<R>? = this@select.getData(name)?.castOrNull(type)
|
||||
|
||||
override val updates: Flow<Name> = this@select.updates.filter {
|
||||
val datum = this@select.getData(it)
|
||||
datum?.type?.isSubtypeOf(type) ?: false
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Select a single datum of the appropriate type
|
||||
*/
|
||||
public inline fun <reified R : Any> DataSet<*>.select(namePattern: Name? = null): DataSet<R> =
|
||||
select(typeOf<R>(), namePattern)
|
||||
|
||||
public suspend fun <R : Any> DataSet<*>.selectOne(type: KType, name: Name): NamedData<R>? =
|
||||
getData(name)?.castOrNull<R>(type)?.named(name)
|
||||
|
||||
public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: Name): NamedData<R>? = selectOne(typeOf<R>(), name)
|
||||
|
||||
public suspend inline fun <reified R : Any> DataSet<*>.selectOne(name: String): NamedData<R>? =
|
||||
selectOne(typeOf<R>(), name.toName())
|
@ -1,42 +1,50 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.test.runTest
|
||||
import org.junit.jupiter.api.Test
|
||||
import space.kscience.dataforge.actions.Action
|
||||
import space.kscience.dataforge.actions.invoke
|
||||
import space.kscience.dataforge.actions.map
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@Suppress("EXPERIMENTAL_API_USAGE")
|
||||
class ActionsTest {
|
||||
val data: DataTree<Int> = runBlocking {
|
||||
DataTree {
|
||||
@OptIn(DFExperimental::class, ExperimentalCoroutinesApi::class)
|
||||
internal class ActionsTest {
|
||||
@Test
|
||||
fun testStaticMapAction() = runTest {
|
||||
val data: DataTree<Int> = DataTree {
|
||||
repeat(10) {
|
||||
static(it.toString(), it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testStaticMapAction() {
|
||||
val plusOne = Action.map<Int, Int> {
|
||||
result { it + 1 }
|
||||
}
|
||||
runBlocking {
|
||||
val result = plusOne.execute(data)
|
||||
assertEquals(2, result.getData("1")?.await())
|
||||
}
|
||||
val result = plusOne(data)
|
||||
assertEquals(2, result["1"]?.await())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testDynamicMapAction() {
|
||||
fun testDynamicMapAction() = runTest {
|
||||
val data: DataSourceBuilder<Int> = DataSource()
|
||||
|
||||
val plusOne = Action.map<Int, Int> {
|
||||
result { it + 1 }
|
||||
}
|
||||
val datum = runBlocking {
|
||||
val result = plusOne.execute(data, scope = this)
|
||||
result.getData("1")?.await()
|
||||
|
||||
val result = plusOne(data)
|
||||
|
||||
repeat(10) {
|
||||
data.static(it.toString(), it)
|
||||
}
|
||||
assertEquals(2, datum)
|
||||
|
||||
delay(20)
|
||||
|
||||
assertEquals(2, result["1"]?.await())
|
||||
data.close()
|
||||
}
|
||||
|
||||
}
|
@ -1,9 +1,8 @@
|
||||
package space.kscience.dataforge.data
|
||||
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.toName
|
||||
import space.kscience.dataforge.names.asName
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -20,10 +19,10 @@ internal class DataTreeBuilderTest {
|
||||
static("c.f", "c.f")
|
||||
}
|
||||
runBlocking {
|
||||
assertEquals("a", node.getData("primary.a")?.await())
|
||||
assertEquals("b", node.getData("primary.b")?.await())
|
||||
assertEquals("c.d", node.getData("c.d")?.await())
|
||||
assertEquals("c.f", node.getData("c.f")?.await())
|
||||
assertEquals("a", node["primary.a"]?.await())
|
||||
assertEquals("b", node["primary.b"]?.await())
|
||||
assertEquals("c.d", node["c.d"]?.await())
|
||||
assertEquals("c.f", node["c.f"]?.await())
|
||||
}
|
||||
}
|
||||
|
||||
@ -43,12 +42,12 @@ internal class DataTreeBuilderTest {
|
||||
static("b", "b")
|
||||
}
|
||||
static("root", "root")
|
||||
populate(updateData)
|
||||
populateFrom(updateData)
|
||||
}
|
||||
|
||||
runBlocking {
|
||||
assertEquals("a", node.getData("update.a")?.await())
|
||||
assertEquals("a", node.getData("primary.a")?.await())
|
||||
assertEquals("a", node["update.a"]?.await())
|
||||
assertEquals("a", node["primary.a"]?.await())
|
||||
}
|
||||
}
|
||||
|
||||
@ -57,7 +56,7 @@ internal class DataTreeBuilderTest {
|
||||
try {
|
||||
lateinit var updateJob: Job
|
||||
supervisorScope {
|
||||
val subNode = ActiveDataTree<Int> {
|
||||
val subNode = DataSource<Int> {
|
||||
updateJob = launch {
|
||||
repeat(10) {
|
||||
delay(10)
|
||||
@ -71,8 +70,8 @@ internal class DataTreeBuilderTest {
|
||||
println(it)
|
||||
}
|
||||
}
|
||||
val rootNode = ActiveDataTree<Int> {
|
||||
setAndObserve("sub".toName(), subNode)
|
||||
val rootNode = DataSource<Int> {
|
||||
setAndWatch("sub".asName(), subNode)
|
||||
}
|
||||
|
||||
launch {
|
||||
@ -81,11 +80,11 @@ internal class DataTreeBuilderTest {
|
||||
}
|
||||
}
|
||||
updateJob.join()
|
||||
assertEquals(9, rootNode.getData("sub.value")?.await())
|
||||
assertEquals(9, rootNode["sub.value"]?.await())
|
||||
cancel()
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
if (t !is CancellationException) throw t
|
||||
if (t !is CancellationException) throw t
|
||||
}
|
||||
|
||||
}
|
||||
|
23
dataforge-io/README.md
Normal file
23
dataforge-io/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-io
|
||||
|
||||
IO module
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-io:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-io:0.7.0")
|
||||
}
|
||||
```
|
@ -1,29 +1,26 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
description = "IO module"
|
||||
|
||||
val ioVersion = "0.2.1"
|
||||
|
||||
kscience {
|
||||
useSerialization(sourceSet = ru.mipt.npm.gradle.DependencySourceSet.TEST) {
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
useSerialization()
|
||||
useSerialization(sourceSet = space.kscience.gradle.DependencySourceSet.TEST) {
|
||||
cbor()
|
||||
}
|
||||
}
|
||||
|
||||
//val ioVersion by rootProject.extra("0.2.0-npm-dev-11")
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
commonMain {
|
||||
dependencies {
|
||||
api(project(":dataforge-context"))
|
||||
api("io.ktor:ktor-io:${ru.mipt.npm.gradle.KScienceVersions.ktorVersion}")
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
api(projects.dataforgeContext)
|
||||
api("org.jetbrains.kotlinx:kotlinx-io-core:$ioVersion")
|
||||
api("org.jetbrains.kotlinx:kotlinx-io-bytestring:$ioVersion")
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE
|
||||
maturity = space.kscience.gradle.Maturity.EXPERIMENTAL
|
||||
}
|
23
dataforge-io/dataforge-io-yaml/README.md
Normal file
23
dataforge-io/dataforge-io-yaml/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-io-yaml
|
||||
|
||||
YAML meta IO
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-io-yaml:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-io-yaml:0.7.0")
|
||||
}
|
||||
```
|
@ -1,32 +1,23 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
// id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
description = "YAML meta IO"
|
||||
|
||||
kscience {
|
||||
useSerialization{
|
||||
yamlKt("0.9.0-dev-1")
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
dependencies {
|
||||
api(projects.dataforgeIo)
|
||||
}
|
||||
}
|
||||
|
||||
repositories{
|
||||
maven("https://dl.bintray.com/mamoe/yamlkt")
|
||||
}
|
||||
|
||||
kotlin {
|
||||
sourceSets {
|
||||
commonMain{
|
||||
dependencies {
|
||||
api(project(":dataforge-io"))
|
||||
}
|
||||
}
|
||||
useSerialization{
|
||||
yamlKt()
|
||||
}
|
||||
}
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.PROTOTYPE
|
||||
maturity = space.kscience.gradle.Maturity.PROTOTYPE
|
||||
description ="""
|
||||
YAML meta converters and Front Matter envelope format
|
||||
""".trimIndent()
|
||||
|
@ -1,123 +1,97 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import io.ktor.utils.io.core.readBytes
|
||||
import io.ktor.utils.io.core.readUTF8Line
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import kotlinx.io.readByteString
|
||||
import kotlinx.io.writeString
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.io.*
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
|
||||
@DFExperimental
|
||||
public class FrontMatterEnvelopeFormat(
|
||||
private val io: IOPlugin,
|
||||
private val meta: Meta = Meta.EMPTY,
|
||||
private val metaFormatFactory: MetaFormatFactory = YamlMetaFormat,
|
||||
) : EnvelopeFormat {
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope {
|
||||
var line: String
|
||||
var offset = 0u
|
||||
do {
|
||||
line = input.readUTF8Line() ?: error("Input does not contain front matter separator")
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
override fun readFrom(binary: Binary): Envelope = binary.read {
|
||||
var offset = 0
|
||||
|
||||
val readMetaFormat =
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
||||
?.let { io.resolveMetaFormat(it) } ?: space.kscience.dataforge.io.yaml.YamlMetaFormat
|
||||
offset += discardWithSeparator(
|
||||
SEPARATOR,
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
//TODO replace by preview
|
||||
val meta = Binary {
|
||||
do {
|
||||
line = input.readSafeUtf8Line()
|
||||
writeUtf8String(line + "\r\n")
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
}.read {
|
||||
readMetaFormat.readMeta(input)
|
||||
val line = ByteArray {
|
||||
offset += readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
}.decodeToString()
|
||||
|
||||
val readMetaFormat = line.trim().takeIf { it.isNotBlank() }?.let { io.resolveMetaFormat(it) } ?: YamlMetaFormat
|
||||
|
||||
val packet = ByteArray {
|
||||
offset += readWithSeparatorTo(this, SEPARATOR)
|
||||
}
|
||||
return PartialEnvelope(meta, offset, null)
|
||||
|
||||
offset += discardLine()
|
||||
|
||||
val meta = readMetaFormat.readFrom(packet.asBinary())
|
||||
Envelope(meta, binary.view(offset))
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope {
|
||||
var line: String
|
||||
do {
|
||||
line = input.readSafeUtf8Line() //?: error("Input does not contain front matter separator")
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
override fun readFrom(source: Source): Envelope = readFrom(source.readBinary())
|
||||
|
||||
val readMetaFormat =
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.metaTypeRegex.matchEntire(line)?.groupValues?.first()
|
||||
?.let { io.resolveMetaFormat(it) } ?: space.kscience.dataforge.io.yaml.YamlMetaFormat
|
||||
|
||||
val meta = Binary {
|
||||
do {
|
||||
writeUtf8String(input.readSafeUtf8Line() + "\r\n")
|
||||
} while (!line.startsWith(space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR))
|
||||
}.read {
|
||||
readMetaFormat.readMeta(input)
|
||||
}
|
||||
val bytes = input.readBytes()
|
||||
val data = bytes.asBinary()
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory(formatMeta, this@FrontMatterEnvelopeFormat.io.context)
|
||||
output.writeRawString("${space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
|
||||
metaFormat.run { this.writeObject(output, envelope.meta) }
|
||||
output.writeRawString("${space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.SEPARATOR}\r\n")
|
||||
val metaFormat = metaFormatFactory.build(io.context, meta)
|
||||
val formatSuffix = if (metaFormat is YamlMetaFormat) "" else metaFormatFactory.shortName
|
||||
sink.writeString("$SEPARATOR${formatSuffix}\r\n")
|
||||
metaFormat.run { metaFormat.writeTo(sink, obj.meta) }
|
||||
sink.writeString("$SEPARATOR\r\n")
|
||||
//Printing data
|
||||
envelope.data?.let { data ->
|
||||
output.writeBinary(data)
|
||||
obj.data?.let { data ->
|
||||
sink.writeBinary(data)
|
||||
}
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
META_KEY put meta
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
public const val SEPARATOR: String = "---"
|
||||
public val SEPARATOR: ByteString = "---".encodeToByteString()
|
||||
|
||||
private val metaTypeRegex = "---(\\w*)\\s*".toRegex()
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
||||
return space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat(context.io, meta)
|
||||
override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "frontMatter"
|
||||
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat {
|
||||
return FrontMatterEnvelopeFormat(context.io, meta)
|
||||
}
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = binary.read {
|
||||
val line = readSafeUtf8Line()
|
||||
return@read if (line.startsWith("---")) {
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.invoke()
|
||||
//read raw string to avoid UTF issues
|
||||
val line = readByteString(3)
|
||||
return@read if (line == "---".encodeToByteString()) {
|
||||
default
|
||||
} else {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private val default by lazy { space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.invoke() }
|
||||
private val default by lazy { build(Global, Meta.EMPTY) }
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope =
|
||||
space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.readPartial(input)
|
||||
override fun readFrom(binary: Binary): Envelope = default.readFrom(binary)
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
): Unit = space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.writeEnvelope(output, envelope, metaFormatFactory, formatMeta)
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
): Unit = default.writeTo(sink, obj)
|
||||
|
||||
|
||||
override fun readObject(input: Input): Envelope = space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.Companion.default.readObject(input)
|
||||
override fun readFrom(source: Source): Envelope = default.readFrom(source)
|
||||
|
||||
}
|
||||
}
|
@ -1,82 +1,84 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readString
|
||||
import kotlinx.io.writeString
|
||||
import net.mamoe.yamlkt.*
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.io.MetaFormat
|
||||
import space.kscience.dataforge.io.MetaFormatFactory
|
||||
import space.kscience.dataforge.io.readUtf8String
|
||||
import space.kscience.dataforge.io.writeUtf8String
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.descriptors.ItemDescriptor
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.descriptors.get
|
||||
import space.kscience.dataforge.names.NameToken
|
||||
import space.kscience.dataforge.names.withIndex
|
||||
import space.kscience.dataforge.values.ListValue
|
||||
import space.kscience.dataforge.values.Null
|
||||
import space.kscience.dataforge.values.parseValue
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.collections.set
|
||||
|
||||
public fun Meta.toYaml(): YamlMap {
|
||||
val map: Map<String, Any?> = items.entries.associate { (key, item) ->
|
||||
key.toString() to when (item) {
|
||||
is MetaItemValue -> {
|
||||
item.value.value
|
||||
}
|
||||
is MetaItemNode -> {
|
||||
item.node.toYaml()
|
||||
}
|
||||
key.toString() to if (item.isLeaf) {
|
||||
item.value?.value
|
||||
} else {
|
||||
item.toYaml()
|
||||
}
|
||||
}
|
||||
|
||||
return YamlMap(map)
|
||||
}
|
||||
|
||||
private class YamlMeta(private val yamlMap: YamlMap, private val descriptor: NodeDescriptor? = null) : MetaBase() {
|
||||
private class YamlMeta(private val yamlMap: YamlMap, private val descriptor: MetaDescriptor? = null) : Meta {
|
||||
|
||||
private fun buildItems(): Map<NameToken, MetaItem> {
|
||||
val map = LinkedHashMap<NameToken, MetaItem>()
|
||||
override val value: Value?
|
||||
get() = yamlMap.getStringOrNull(null)?.let { Value.parse(it) }
|
||||
|
||||
private fun buildItems(): Map<NameToken, Meta> {
|
||||
val map = LinkedHashMap<NameToken, Meta>()
|
||||
|
||||
yamlMap.content.entries.forEach { (key, value) ->
|
||||
val stringKey = key.toString()
|
||||
val itemDescriptor = descriptor?.items?.get(stringKey)
|
||||
val itemDescriptor = descriptor?.get(stringKey)
|
||||
val token = NameToken(stringKey)
|
||||
when (value) {
|
||||
YamlNull -> Null.asMetaItem()
|
||||
is YamlLiteral -> map[token] = value.content.parseValue().asMetaItem()
|
||||
is YamlMap -> map[token] = value.toMeta().asMetaItem()
|
||||
YamlNull -> Meta(Null)
|
||||
is YamlLiteral -> map[token] = Meta(Value.parse(value.content))
|
||||
is YamlMap -> map[token] = value.toMeta()
|
||||
is YamlList -> if (value.all { it is YamlLiteral }) {
|
||||
val listValue = ListValue(
|
||||
value.map {
|
||||
//We already checked that all values are primitives
|
||||
(it as YamlLiteral).content.parseValue()
|
||||
Value.parse((it as YamlLiteral).content)
|
||||
}
|
||||
)
|
||||
map[token] = MetaItemValue(listValue)
|
||||
map[token] = Meta(listValue)
|
||||
} else value.forEachIndexed { index, yamlElement ->
|
||||
val indexKey = (itemDescriptor as? NodeDescriptor)?.indexKey ?: ItemDescriptor.DEFAULT_INDEX_KEY
|
||||
val indexKey = itemDescriptor?.indexKey
|
||||
val indexValue: String = (yamlElement as? YamlMap)?.getStringOrNull(indexKey)
|
||||
?: index.toString() //In case index is non-string, the backward transformation will be broken.
|
||||
|
||||
val tokenWithIndex = token.withIndex(indexValue)
|
||||
map[tokenWithIndex] = yamlElement.toMetaItem(itemDescriptor)
|
||||
map[tokenWithIndex] = yamlElement.toMeta(itemDescriptor)
|
||||
}
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
override val items: Map<NameToken, MetaItem> get() = buildItems()
|
||||
override val items: Map<NameToken, Meta> get() = buildItems()
|
||||
|
||||
override fun toString(): String = Meta.toString(this)
|
||||
override fun equals(other: Any?): Boolean = Meta.equals(this, other as? Meta)
|
||||
override fun hashCode(): Int = Meta.hashCode(this)
|
||||
}
|
||||
|
||||
public fun YamlElement.toMetaItem(descriptor: ItemDescriptor? = null): MetaItem = when (this) {
|
||||
YamlNull -> Null.asMetaItem()
|
||||
is YamlLiteral -> content.parseValue().asMetaItem()
|
||||
is YamlMap -> toMeta().asMetaItem()
|
||||
public fun YamlElement.toMeta(descriptor: MetaDescriptor? = null): Meta = when (this) {
|
||||
YamlNull -> Meta(Null)
|
||||
is YamlLiteral -> Meta(Value.parse(content))
|
||||
is YamlMap -> toMeta()
|
||||
//We can't return multiple items therefore we create top level node
|
||||
is YamlList -> YamlMap(mapOf("@yamlArray" to this)).toMetaItem(descriptor)
|
||||
is YamlList -> YamlMap(mapOf("@yamlArray" to this)).toMeta(descriptor)
|
||||
}
|
||||
|
||||
public fun YamlMap.toMeta(): Meta = YamlMeta(this)
|
||||
@ -85,38 +87,32 @@ public fun YamlMap.toMeta(): Meta = YamlMeta(this)
|
||||
/**
|
||||
* Represent meta as Yaml
|
||||
*/
|
||||
@DFExperimental
|
||||
public class YamlMetaFormat(private val meta: Meta) : MetaFormat {
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) {
|
||||
val yaml = meta.toYaml()
|
||||
val string = Yaml.encodeToString(yaml)
|
||||
output.writeUtf8String(string)
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
val yaml: YamlMap = meta.toYaml()
|
||||
val string = Yaml.encodeToString(YamlMap.serializer(), yaml)
|
||||
sink.writeString(string)
|
||||
}
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
|
||||
val yaml = Yaml.decodeYamlMapFromString(input.readUtf8String())
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta {
|
||||
val yaml = Yaml.decodeYamlMapFromString(source.readString())
|
||||
return yaml.toMeta()
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put space.kscience.dataforge.io.yaml.FrontMatterEnvelopeFormat.name.toString()
|
||||
META_KEY put meta
|
||||
}
|
||||
|
||||
public companion object : MetaFormatFactory {
|
||||
override fun invoke(meta: Meta, context: Context): MetaFormat = YamlMetaFormat(meta)
|
||||
override fun build(context: Context, meta: Meta): MetaFormat = YamlMetaFormat(meta)
|
||||
|
||||
override val shortName: String = "yaml"
|
||||
|
||||
override val key: Short = 0x594d //YM
|
||||
|
||||
private val default = YamlMetaFormat()
|
||||
private val default = YamlMetaFormat(Meta.EMPTY)
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit =
|
||||
default.writeMeta(output, meta, descriptor)
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.writeMeta(sink, meta, descriptor)
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta =
|
||||
default.readMeta(input, descriptor)
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
default.readMeta(source, descriptor)
|
||||
}
|
||||
}
|
@ -0,0 +1,30 @@
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import space.kscience.dataforge.context.AbstractPlugin
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.PluginFactory
|
||||
import space.kscience.dataforge.context.PluginTag
|
||||
import space.kscience.dataforge.io.EnvelopeFormatFactory
|
||||
import space.kscience.dataforge.io.IOPlugin
|
||||
import space.kscience.dataforge.io.MetaFormatFactory
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
|
||||
public class YamlPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
public val io: IOPlugin by require(IOPlugin)
|
||||
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
|
||||
override fun content(target: String): Map<Name, Any> = when (target) {
|
||||
MetaFormatFactory.META_FORMAT_TYPE -> mapOf("yaml".asName() to YamlMetaFormat)
|
||||
EnvelopeFormatFactory.ENVELOPE_FORMAT_TYPE -> mapOf(FrontMatterEnvelopeFormat.name to FrontMatterEnvelopeFormat)
|
||||
else -> super.content(target)
|
||||
}
|
||||
|
||||
public companion object : PluginFactory<YamlPlugin> {
|
||||
override val tag: PluginTag = PluginTag("io.yaml", group = PluginTag.DATAFORGE_GROUP)
|
||||
|
||||
override fun build(context: Context, meta: Meta): YamlPlugin = YamlPlugin(meta)
|
||||
}
|
||||
}
|
@ -0,0 +1,37 @@
|
||||
@file:OptIn(DFExperimental::class)
|
||||
|
||||
package space.kscience.dataforge.io.yaml
|
||||
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.io
|
||||
import space.kscience.dataforge.io.readEnvelope
|
||||
import space.kscience.dataforge.io.toByteArray
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
internal class FrontMatterEnvelopeFormatTest {
|
||||
|
||||
val context = Context {
|
||||
plugin(YamlPlugin)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun frontMatter(){
|
||||
val text = """
|
||||
---
|
||||
content_type: magprog
|
||||
magprog_section: contacts
|
||||
section_title: Контакты
|
||||
language: ru
|
||||
---
|
||||
Some text here
|
||||
""".trimIndent()
|
||||
|
||||
val envelope = context.io.readEnvelope(text)
|
||||
assertEquals("Some text here", envelope.data!!.toByteArray().decodeToString().trim())
|
||||
assertEquals("magprog", envelope.meta["content_type"].string)
|
||||
}
|
||||
}
|
@ -1,6 +1,9 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.buffered
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlin.math.min
|
||||
|
||||
/**
|
||||
@ -12,11 +15,20 @@ public interface Binary {
|
||||
|
||||
public val size: Int
|
||||
|
||||
|
||||
/**
|
||||
* Read maximum of [atMost] bytes as input from the binary, starting at [offset]. The generated input is always closed
|
||||
* when leaving scope, so it could not be leaked outside of scope of [block].
|
||||
*/
|
||||
public fun <R> read(offset: Int = 0, atMost: Int = size - offset, block: Input.() -> R): R
|
||||
public fun <R> read(offset: Int = 0, atMost: Int = size - offset, block: Source.() -> R): R
|
||||
|
||||
public suspend fun <R> readSuspend(offset: Int = 0, atMost: Int = size - offset, block: suspend Source.() -> R): R
|
||||
|
||||
/**
|
||||
* Read a binary with given [offset] relative to this binary and given [binarySize].
|
||||
* In general, the resulting binary is of the same type as this one, but it is not guaranteed.
|
||||
*/
|
||||
public fun view(offset: Int, binarySize: Int = size - offset): Binary
|
||||
|
||||
public companion object {
|
||||
public val EMPTY: Binary = ByteArrayBinary(ByteArray(0))
|
||||
@ -29,46 +41,67 @@ internal class ByteArrayBinary(
|
||||
override val size: Int = array.size - start,
|
||||
) : Binary {
|
||||
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Source.() -> R): R {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset < array.size) { "Offset $offset is larger than array size" }
|
||||
val input = ByteReadPacket(
|
||||
|
||||
return ByteArraySource(
|
||||
array,
|
||||
offset + start,
|
||||
min(atMost, size - offset)
|
||||
)
|
||||
return input.use(block)
|
||||
).buffered().use(block)
|
||||
}
|
||||
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset < array.size) { "Offset $offset is larger than array size" }
|
||||
|
||||
val input = ByteArraySource(
|
||||
array,
|
||||
offset + start,
|
||||
min(atMost, size - offset)
|
||||
).buffered()
|
||||
|
||||
return try {
|
||||
block(input)
|
||||
} finally {
|
||||
input.close()
|
||||
}
|
||||
}
|
||||
|
||||
override fun view(offset: Int, binarySize: Int): ByteArrayBinary =
|
||||
ByteArrayBinary(array, start + offset, binarySize)
|
||||
}
|
||||
|
||||
public fun ByteArray.asBinary(): Binary = ByteArrayBinary(this)
|
||||
|
||||
/**
|
||||
* Produce a [buildByteArray] representing an exact copy of this [Binary]
|
||||
* Produce a [ByteArray] representing an exact copy of this [Binary]
|
||||
*/
|
||||
public fun Binary.toByteArray(): ByteArray = if (this is ByteArrayBinary) {
|
||||
array.copyOf() // TODO do we need to ensure data safety here?
|
||||
array.copyOfRange(start, start + size) // TODO do we need to ensure data safety here?
|
||||
} else {
|
||||
read {
|
||||
readBytes()
|
||||
readByteArray()
|
||||
}
|
||||
}
|
||||
|
||||
public fun Input.readBinary(size: Int): Binary {
|
||||
val array = readBytes(size)
|
||||
//TODO optimize for file-based Inputs
|
||||
public fun Source.readBinary(size: Int? = null): Binary {
|
||||
val array = if (size == null) readByteArray() else readByteArray(size)
|
||||
return ByteArrayBinary(array)
|
||||
}
|
||||
|
||||
/**
|
||||
* Direct write of binary to the output. Returns the number of bytes written
|
||||
*/
|
||||
public fun Output.writeBinary(binary: Binary): Int {
|
||||
public fun Sink.writeBinary(binary: Binary): Int {
|
||||
return if (binary is ByteArrayBinary) {
|
||||
writeFully(binary.array, binary.start, binary.start + binary.size)
|
||||
write(binary.array, binary.start, binary.start + binary.size)
|
||||
binary.size
|
||||
} else {
|
||||
binary.read {
|
||||
copyTo(this@writeBinary).toInt()
|
||||
transferTo(this@writeBinary).toInt()
|
||||
}
|
||||
}
|
||||
}
|
@ -1,133 +0,0 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.values.*
|
||||
|
||||
/**
|
||||
* A DataForge-specific simplified binary format for meta
|
||||
* TODO add description
|
||||
*/
|
||||
public object BinaryMetaFormat : MetaFormat, MetaFormatFactory {
|
||||
override val shortName: String = "bin"
|
||||
override val key: Short = 0x4249//BI
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): MetaFormat = this
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
|
||||
return (input.readMetaItem() as MetaItemNode).node
|
||||
}
|
||||
|
||||
private fun Output.writeChar(char: Char) = writeByte(char.code.toByte())
|
||||
|
||||
private fun Output.writeString(str: String) {
|
||||
writeInt(str.length)
|
||||
writeFully(str.encodeToByteArray())
|
||||
}
|
||||
|
||||
public fun Output.writeValue(value: Value): Unit = when (value.type) {
|
||||
ValueType.NUMBER -> when (value.value) {
|
||||
is Short -> {
|
||||
writeChar('s')
|
||||
writeShort(value.short)
|
||||
}
|
||||
is Int -> {
|
||||
writeChar('i')
|
||||
writeInt(value.int)
|
||||
}
|
||||
is Long -> {
|
||||
writeChar('l')
|
||||
writeLong(value.long)
|
||||
}
|
||||
is Float -> {
|
||||
writeChar('f')
|
||||
writeFloat(value.float)
|
||||
}
|
||||
else -> {
|
||||
writeChar('d')
|
||||
writeDouble(value.double)
|
||||
}
|
||||
}
|
||||
ValueType.STRING -> {
|
||||
writeChar('S')
|
||||
writeString(value.string)
|
||||
}
|
||||
ValueType.BOOLEAN -> {
|
||||
if (value.boolean) {
|
||||
writeChar('+')
|
||||
} else {
|
||||
writeChar('-')
|
||||
}
|
||||
}
|
||||
ValueType.NULL -> {
|
||||
writeChar('N')
|
||||
}
|
||||
ValueType.LIST -> {
|
||||
writeChar('L')
|
||||
writeInt(value.list.size)
|
||||
value.list.forEach {
|
||||
writeValue(it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun writeMeta(
|
||||
output: Output,
|
||||
meta: Meta,
|
||||
descriptor: space.kscience.dataforge.meta.descriptors.NodeDescriptor?,
|
||||
) {
|
||||
output.writeChar('M')
|
||||
output.writeInt(meta.items.size)
|
||||
meta.items.forEach { (key, item) ->
|
||||
output.writeString(key.toString())
|
||||
when (item) {
|
||||
is MetaItemValue -> {
|
||||
output.writeValue(item.value)
|
||||
}
|
||||
is MetaItemNode -> {
|
||||
writeObject(output, item.node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun Input.readString(): String {
|
||||
val length = readInt()
|
||||
val array = readBytes(length)
|
||||
return array.decodeToString()
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
public fun Input.readMetaItem(): TypedMetaItem<MetaBuilder> {
|
||||
return when (val keyChar = readByte().toInt().toChar()) {
|
||||
'S' -> MetaItemValue(StringValue(readString()))
|
||||
'N' -> MetaItemValue(Null)
|
||||
'+' -> MetaItemValue(True)
|
||||
'-' -> MetaItemValue(True)
|
||||
's' -> MetaItemValue(NumberValue(readShort()))
|
||||
'i' -> MetaItemValue(NumberValue(readInt()))
|
||||
'l' -> MetaItemValue(NumberValue(readInt()))
|
||||
'f' -> MetaItemValue(NumberValue(readFloat()))
|
||||
'd' -> MetaItemValue(NumberValue(readDouble()))
|
||||
'L' -> {
|
||||
val length = readInt()
|
||||
val list = (1..length).map { (readMetaItem() as MetaItemValue).value }
|
||||
MetaItemValue(Value.of(list))
|
||||
}
|
||||
'M' -> {
|
||||
val length = readInt()
|
||||
val meta = Meta {
|
||||
(1..length).forEach { _ ->
|
||||
val name = readString()
|
||||
val item = readMetaItem()
|
||||
set(name, item)
|
||||
}
|
||||
}
|
||||
MetaItemNode(meta)
|
||||
}
|
||||
else -> error("Unknown serialization key character: $keyChar")
|
||||
}
|
||||
}
|
||||
}
|
@ -34,7 +34,9 @@ public interface Envelope {
|
||||
}
|
||||
}
|
||||
|
||||
public class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
|
||||
internal class SimpleEnvelope(override val meta: Meta, override val data: Binary?) : Envelope
|
||||
|
||||
public fun Envelope(meta: Meta, data: Binary?): Envelope = SimpleEnvelope(meta, data)
|
||||
|
||||
/**
|
||||
* The purpose of the envelope
|
||||
|
@ -1,10 +1,10 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Sink
|
||||
import space.kscience.dataforge.meta.*
|
||||
|
||||
public class EnvelopeBuilder : Envelope {
|
||||
private val metaBuilder = MetaBuilder()
|
||||
private val metaBuilder = MutableMeta()
|
||||
|
||||
override var data: Binary? = null
|
||||
override var meta: Meta
|
||||
@ -13,7 +13,7 @@ public class EnvelopeBuilder : Envelope {
|
||||
metaBuilder.update(value)
|
||||
}
|
||||
|
||||
public fun meta(block: MetaBuilder.() -> Unit) {
|
||||
public fun meta(block: MutableMeta.() -> Unit) {
|
||||
metaBuilder.apply(block)
|
||||
}
|
||||
|
||||
@ -33,8 +33,8 @@ public class EnvelopeBuilder : Envelope {
|
||||
/**
|
||||
* Construct a data binary from given builder
|
||||
*/
|
||||
public fun data(block: Output.() -> Unit) {
|
||||
data = buildByteArray { block() }.asBinary()
|
||||
public inline fun data(block: Sink.() -> Unit) {
|
||||
data = ByteArray { block() }.asBinary()
|
||||
}
|
||||
|
||||
public fun seal(): Envelope = SimpleEnvelope(metaBuilder.seal(), data)
|
||||
|
@ -1,48 +1,27 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Source
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
/**
|
||||
* A partially read envelope with meta, but without data
|
||||
*/
|
||||
public data class PartialEnvelope(val meta: Meta, val dataOffset: UInt, val dataSize: ULong?)
|
||||
|
||||
public interface EnvelopeFormat : IOFormat<Envelope> {
|
||||
|
||||
override val type: KType get() = typeOf<Envelope>()
|
||||
|
||||
public val defaultMetaFormat: MetaFormatFactory get() = JsonMetaFormat
|
||||
|
||||
public fun readPartial(input: Input): PartialEnvelope
|
||||
|
||||
public fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory = defaultMetaFormat,
|
||||
formatMeta: Meta = Meta.EMPTY,
|
||||
)
|
||||
|
||||
override fun readObject(input: Input): Envelope
|
||||
|
||||
override fun writeObject(output: Output, obj: Envelope): Unit = writeEnvelope(output, obj)
|
||||
}
|
||||
|
||||
public fun EnvelopeFormat.read(input: Input): Envelope = readObject(input)
|
||||
public fun EnvelopeFormat.read(input: Source): Envelope = readFrom(input)
|
||||
|
||||
@Type(ENVELOPE_FORMAT_TYPE)
|
||||
@DfId(ENVELOPE_FORMAT_TYPE)
|
||||
public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeFormat {
|
||||
override val name: Name get() = "envelope".asName()
|
||||
override val type: KType get() = typeOf<Envelope>()
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat
|
||||
|
||||
/**
|
||||
* Try to infer specific format from input and return null if the attempt is failed.
|
||||
@ -51,6 +30,7 @@ public interface EnvelopeFormatFactory : IOFormatFactory<Envelope>, EnvelopeForm
|
||||
public fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat?
|
||||
|
||||
public companion object {
|
||||
public val ENVELOPE_FACTORY_NAME: Name = "envelope".asName()
|
||||
public const val ENVELOPE_FORMAT_TYPE: String = "io.format.envelope"
|
||||
}
|
||||
}
|
@ -1,30 +1,29 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import kotlinx.io.write
|
||||
import space.kscience.dataforge.io.Envelope.Companion.ENVELOPE_NODE_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.DEFAULT_MULTIPART_DATA_SEPARATOR
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_DATA_TYPE
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.MULTIPART_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.PARTS_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.PART_FORMAT_KEY
|
||||
import space.kscience.dataforge.io.PartDescriptor.Companion.SEPARATOR_KEY
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.plus
|
||||
import space.kscience.dataforge.names.toName
|
||||
|
||||
private class PartDescriptor : Scheme() {
|
||||
var offset by int(0)
|
||||
var size by int(0)
|
||||
var partMeta by node("meta".toName())
|
||||
var partMeta by node("meta".asName())
|
||||
|
||||
companion object : SchemeSpec<PartDescriptor>(::PartDescriptor) {
|
||||
val MULTIPART_KEY = ENVELOPE_NODE_KEY + "multipart"
|
||||
val PARTS_KEY = MULTIPART_KEY + "parts"
|
||||
val SEPARATOR_KEY = MULTIPART_KEY + "separator"
|
||||
|
||||
const val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n"
|
||||
|
||||
val PART_FORMAT_KEY = "format".asName()
|
||||
val DEFAULT_MULTIPART_DATA_SEPARATOR = "\r\n#~PART~#\r\n".toAsciiByteString()
|
||||
|
||||
const val MULTIPART_DATA_TYPE = "envelope.multipart"
|
||||
}
|
||||
@ -36,12 +35,12 @@ public typealias EnvelopeParts = List<EnvelopePart>
|
||||
|
||||
public fun EnvelopeBuilder.multipart(
|
||||
parts: EnvelopeParts,
|
||||
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
|
||||
separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
) {
|
||||
dataType = MULTIPART_DATA_TYPE
|
||||
|
||||
var offsetCounter = 0
|
||||
val separatorSize = separator.length
|
||||
val separatorSize = separator.size
|
||||
val partDescriptors = parts.map { (binary, description) ->
|
||||
offsetCounter += separatorSize
|
||||
PartDescriptor {
|
||||
@ -55,46 +54,45 @@ public fun EnvelopeBuilder.multipart(
|
||||
|
||||
meta {
|
||||
if (separator != DEFAULT_MULTIPART_DATA_SEPARATOR) {
|
||||
SEPARATOR_KEY put separator
|
||||
SEPARATOR_KEY put separator.decodeToString()
|
||||
}
|
||||
setIndexed(PARTS_KEY, partDescriptors.map { it.toMeta() })
|
||||
}
|
||||
|
||||
data {
|
||||
parts.forEach {
|
||||
writeRawString(separator)
|
||||
write(separator)
|
||||
writeBinary(it.binary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a list of envelopes as parts of given envelope
|
||||
*/
|
||||
public fun EnvelopeBuilder.envelopes(
|
||||
envelopes: List<Envelope>,
|
||||
format: EnvelopeFormat = TaggedEnvelopeFormat,
|
||||
separator: String = DEFAULT_MULTIPART_DATA_SEPARATOR
|
||||
separator: ByteString = DEFAULT_MULTIPART_DATA_SEPARATOR,
|
||||
) {
|
||||
val parts = envelopes.map {
|
||||
val binary = format.toBinary(it)
|
||||
val binary = Binary(it, TaggedEnvelopeFormat)
|
||||
EnvelopePart(binary, null)
|
||||
}
|
||||
meta{
|
||||
set(MULTIPART_KEY + PART_FORMAT_KEY, format.toMeta())
|
||||
}
|
||||
multipart(parts, separator)
|
||||
}
|
||||
|
||||
public fun Envelope.parts(): EnvelopeParts {
|
||||
if (data == null) return emptyList()
|
||||
//TODO add zip folder reader
|
||||
val parts = meta.getIndexed(PARTS_KEY).values.mapNotNull { it.node }.map {
|
||||
val parts = meta.getIndexed(PARTS_KEY).values.map {
|
||||
PartDescriptor.read(it)
|
||||
}
|
||||
return if (parts.isEmpty()) {
|
||||
listOf(EnvelopePart(data!!, meta[MULTIPART_KEY].node))
|
||||
listOf(EnvelopePart(data!!, meta[MULTIPART_KEY]))
|
||||
} else {
|
||||
parts.map {
|
||||
val binary = data!!.view(it.offset, it.size)
|
||||
val meta = Laminate(it.partMeta, meta[MULTIPART_KEY].node)
|
||||
val meta = Laminate(it.partMeta, meta[MULTIPART_KEY])
|
||||
EnvelopePart(binary, meta)
|
||||
}
|
||||
}
|
||||
@ -107,14 +105,4 @@ public val EnvelopePart.name: String? get() = description?.get("name").string
|
||||
/**
|
||||
* Represent envelope part by an envelope
|
||||
*/
|
||||
public fun EnvelopePart.envelope(plugin: IOPlugin): Envelope {
|
||||
val formatItem = description?.get(PART_FORMAT_KEY)
|
||||
return if (formatItem != null) {
|
||||
val format: EnvelopeFormat = plugin.resolveEnvelopeFormat(formatItem)
|
||||
?: error("Envelope format for $formatItem is not resolved")
|
||||
binary.readWith(format)
|
||||
} else {
|
||||
error("Envelope description not found")
|
||||
//SimpleEnvelope(description ?: Meta.EMPTY, binary)
|
||||
}
|
||||
}
|
||||
public fun EnvelopePart.envelope(): Envelope = binary.readWith(TaggedEnvelopeFormat)
|
@ -1,135 +1,118 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readByteArray
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Factory
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaItemValue
|
||||
import space.kscience.dataforge.meta.MetaRepr
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.misc.Named
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.values.Value
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
/**
|
||||
* And interface for reading and writing objects into with IO streams
|
||||
* Reader of a custom object from input
|
||||
*/
|
||||
public interface IOFormat<T : Any> : MetaRepr {
|
||||
public interface IOReader<out T> {
|
||||
/**
|
||||
* The type of object being read
|
||||
*/
|
||||
public val type: KType
|
||||
|
||||
public fun writeObject(output: Output, obj: T)
|
||||
public fun readObject(input: Input): T
|
||||
public fun readFrom(source: Source): T
|
||||
|
||||
public fun readFrom(binary: Binary): T = binary.read { readFrom(this) }
|
||||
|
||||
public companion object {
|
||||
public val NAME_KEY: Name = "name".asName()
|
||||
public val META_KEY: Name = "meta".asName()
|
||||
/**
|
||||
* no-op reader for binaries.
|
||||
*/
|
||||
public val binary: IOReader<Binary> = object : IOReader<Binary> {
|
||||
override val type: KType = typeOf<Binary>()
|
||||
|
||||
override fun readFrom(source: Source): Binary = source.readByteArray().asBinary()
|
||||
|
||||
override fun readFrom(binary: Binary): Binary = binary
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public fun <T : Any> Input.readWith(format: IOFormat<T>): T = format.readObject(this@readWith)
|
||||
public inline fun <reified T> IOReader(crossinline read: Source.() -> T): IOReader<T> = object : IOReader<T> {
|
||||
override val type: KType = typeOf<T>()
|
||||
|
||||
public fun <T: Any> IOFormat<T>.readObject(binary: Binary): T = binary.read {
|
||||
readObject(this)
|
||||
override fun readFrom(source: Source): T = source.read()
|
||||
}
|
||||
|
||||
public fun interface IOWriter<in T> {
|
||||
public fun writeTo(sink: Sink, obj: T)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read given binary as object using given format
|
||||
* And interface for reading and writing objects into with IO streams
|
||||
*/
|
||||
public fun <T : Any> Binary.readWith(format: IOFormat<T>): T = read {
|
||||
public interface IOFormat<T> : IOReader<T>, IOWriter<T>
|
||||
|
||||
public fun <T : Any> Source.readWith(format: IOReader<T>): T = format.readFrom(this)
|
||||
|
||||
/**
|
||||
* Read given binary as an object using given format
|
||||
*/
|
||||
public fun <T : Any> Binary.readWith(format: IOReader<T>): T = read {
|
||||
readWith(format)
|
||||
}
|
||||
|
||||
public fun <T : Any> Output.writeWith(format: IOFormat<T>, obj: T): Unit =
|
||||
format.run { writeObject(this@writeWith, obj) }
|
||||
/**
|
||||
* Write an object to the [Sink] with given [format]
|
||||
*/
|
||||
public fun <T : Any> Sink.writeWith(format: IOWriter<T>, obj: T): Unit =
|
||||
format.writeTo(this, obj)
|
||||
|
||||
public inline fun <reified T : Any> IOFormat.Companion.listOf(
|
||||
format: IOFormat<T>,
|
||||
): IOFormat<List<T>> = object : IOFormat<List<T>> {
|
||||
override val type: KType = typeOf<List<T>>()
|
||||
|
||||
override fun writeObject(output: Output, obj: List<T>) {
|
||||
output.writeInt(obj.size)
|
||||
format.run {
|
||||
obj.forEach {
|
||||
writeObject(output, it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): List<T> {
|
||||
val size = input.readInt()
|
||||
return format.run {
|
||||
List(size) { readObject(input) }
|
||||
}
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put "list"
|
||||
"contentFormat" put format.toMeta()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//public fun ObjectPool<Buffer>.fill(block: Buffer.() -> Unit): Buffer {
|
||||
// val buffer = borrow()
|
||||
// return try {
|
||||
// buffer.apply(block)
|
||||
// } catch (ex: Exception) {
|
||||
// //recycle(buffer)
|
||||
// throw ex
|
||||
// }
|
||||
//}
|
||||
|
||||
@Type(IO_FORMAT_TYPE)
|
||||
public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named, MetaRepr {
|
||||
@DfId(IO_FORMAT_TYPE)
|
||||
public interface IOFormatFactory<T : Any> : Factory<IOFormat<T>>, Named {
|
||||
/**
|
||||
* Explicit type for dynamic type checks
|
||||
*/
|
||||
public val type: KType
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
}
|
||||
|
||||
public companion object {
|
||||
public const val IO_FORMAT_TYPE: String = "io.format"
|
||||
public val NAME_KEY: Name = "name".asName()
|
||||
public val META_KEY: Name = "meta".asName()
|
||||
}
|
||||
}
|
||||
|
||||
public fun <T : Any> IOFormat<T>.toBinary(obj: T): Binary = Binary { writeObject(this, obj) }
|
||||
public fun <T : Any> Binary(obj: T, format: IOWriter<T>): Binary = Binary { format.writeTo(this, obj) }
|
||||
|
||||
public object FloatIOFormat : IOFormat<Float>, IOFormatFactory<Float> {
|
||||
override fun build(context: Context, meta: Meta): IOFormat<Float> = this
|
||||
|
||||
override val name: Name = "float32".asName()
|
||||
|
||||
override val type: KType get() = typeOf<Float>()
|
||||
|
||||
override fun writeTo(sink: Sink, obj: Float) {
|
||||
sink.writeFloat(obj)
|
||||
}
|
||||
|
||||
override fun readFrom(source: Source): Float = source.readFloat()
|
||||
}
|
||||
|
||||
|
||||
public object DoubleIOFormat : IOFormat<Double>, IOFormatFactory<Double> {
|
||||
override fun invoke(meta: Meta, context: Context): IOFormat<Double> = this
|
||||
override fun build(context: Context, meta: Meta): IOFormat<Double> = this
|
||||
|
||||
override val name: Name = "double".asName()
|
||||
override val name: Name = "float64".asName()
|
||||
|
||||
override val type: KType get() = typeOf<Double>()
|
||||
|
||||
override fun writeObject(output: Output, obj: kotlin.Double) {
|
||||
output.writeDouble(obj)
|
||||
override fun writeTo(sink: Sink, obj: Double) {
|
||||
sink.writeLong(obj.toBits())
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Double = input.readDouble()
|
||||
}
|
||||
|
||||
public object ValueIOFormat : IOFormat<Value>, IOFormatFactory<Value> {
|
||||
override fun invoke(meta: Meta, context: Context): IOFormat<Value> = this
|
||||
|
||||
override val name: Name = "value".asName()
|
||||
|
||||
override val type: KType get() = typeOf<Value>()
|
||||
|
||||
override fun writeObject(output: Output, obj: Value) {
|
||||
BinaryMetaFormat.run { output.writeValue(obj) }
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Value {
|
||||
return (BinaryMetaFormat.run { input.readMetaItem() } as? MetaItemValue)?.value
|
||||
?: error("The item is not a value")
|
||||
}
|
||||
override fun readFrom(source: Source): Double = source.readDouble()
|
||||
}
|
@ -2,15 +2,15 @@ package space.kscience.dataforge.io
|
||||
|
||||
import space.kscience.dataforge.context.*
|
||||
import space.kscience.dataforge.io.EnvelopeFormatFactory.Companion.ENVELOPE_FORMAT_TYPE
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.io.IOFormatFactory.Companion.IO_FORMAT_TYPE
|
||||
import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.misc.DFInternal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.native.concurrent.ThreadLocal
|
||||
import kotlin.reflect.KClass
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
|
||||
public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
override val tag: PluginTag get() = Companion.tag
|
||||
@ -19,15 +19,14 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
context.gather<IOFormatFactory<*>>(IO_FORMAT_TYPE).values
|
||||
}
|
||||
|
||||
public fun <T : Any> resolveIOFormat(item: MetaItem, type: KClass<out T>): IOFormat<T>? {
|
||||
val key = item.string ?: item.node[NAME_KEY]?.string ?: error("Format name not defined")
|
||||
val name = key.toName()
|
||||
return ioFormatFactories.find { it.name == name }?.let {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
if (it.type != type) error("Format type ${it.type} is not the same as requested type $type")
|
||||
else it.invoke(item.node[META_KEY].node ?: Meta.EMPTY, context) as IOFormat<T>
|
||||
}
|
||||
}
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
@DFInternal
|
||||
public fun <T : Any> resolveIOFormat(type: KType, meta: Meta): IOFormat<T>? =
|
||||
ioFormatFactories.singleOrNull { it.type == type }?.build(context, meta) as? IOFormat<T>
|
||||
|
||||
@OptIn(DFInternal::class)
|
||||
public inline fun <reified T : Any> resolveIOFormat(meta: Meta = Meta.EMPTY): IOFormat<T>? =
|
||||
resolveIOFormat(typeOf<T>(), meta)
|
||||
|
||||
|
||||
public val metaFormatFactories: Collection<MetaFormatFactory> by lazy {
|
||||
@ -35,48 +34,53 @@ public class IOPlugin(meta: Meta) : AbstractPlugin(meta) {
|
||||
}
|
||||
|
||||
public fun resolveMetaFormat(key: Short, meta: Meta = Meta.EMPTY): MetaFormat? =
|
||||
metaFormatFactories.find { it.key == key }?.invoke(meta)
|
||||
metaFormatFactories.find { it.key == key }?.build(context, meta)
|
||||
|
||||
public fun resolveMetaFormat(name: String, meta: Meta = Meta.EMPTY): MetaFormat? =
|
||||
metaFormatFactories.find { it.shortName == name }?.invoke(meta)
|
||||
metaFormatFactories.find { it.shortName == name }?.build(context, meta)
|
||||
|
||||
public val envelopeFormatFactories: Collection<EnvelopeFormatFactory> by lazy {
|
||||
context.gather<EnvelopeFormatFactory>(ENVELOPE_FORMAT_TYPE).values
|
||||
}
|
||||
|
||||
private fun resolveEnvelopeFormat(name: Name, meta: Meta = Meta.EMPTY): EnvelopeFormat? =
|
||||
envelopeFormatFactories.find { it.name == name }?.invoke(meta, context)
|
||||
envelopeFormatFactories.find { it.name == name }?.build(context, meta)
|
||||
|
||||
public fun resolveEnvelopeFormat(item: MetaItem): EnvelopeFormat? {
|
||||
val name = item.string ?: item.node[NAME_KEY]?.string ?: error("Envelope format name not defined")
|
||||
val meta = item.node[META_KEY].node ?: Meta.EMPTY
|
||||
return resolveEnvelopeFormat(name.toName(), meta)
|
||||
public fun resolveEnvelopeFormat(item: Meta): EnvelopeFormat? {
|
||||
val name = item.string ?: item[IOFormatFactory.NAME_KEY]?.string ?: error("Envelope format name not defined")
|
||||
val meta = item[IOFormatFactory.META_KEY] ?: Meta.EMPTY
|
||||
return resolveEnvelopeFormat(Name.parse(name), meta)
|
||||
}
|
||||
|
||||
override fun content(target: String): Map<Name, Any> {
|
||||
return when (target) {
|
||||
META_FORMAT_TYPE -> defaultMetaFormats.toMap()
|
||||
ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.toMap()
|
||||
else -> super.content(target)
|
||||
}
|
||||
override fun content(target: String): Map<Name, Any> = when (target) {
|
||||
META_FORMAT_TYPE -> defaultMetaFormats.associateByName()
|
||||
ENVELOPE_FORMAT_TYPE -> defaultEnvelopeFormats.associateByName()
|
||||
IO_FORMAT_TYPE -> content(META_FORMAT_TYPE) + content(ENVELOPE_FORMAT_TYPE)
|
||||
else -> super.content(target)
|
||||
}
|
||||
|
||||
public companion object : PluginFactory<IOPlugin> {
|
||||
public val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat, BinaryMetaFormat)
|
||||
public val defaultEnvelopeFormats: List<EnvelopeFormatFactory> =
|
||||
listOf(TaggedEnvelopeFormat, TaglessEnvelopeFormat)
|
||||
public val defaultMetaFormats: List<MetaFormatFactory> = listOf(JsonMetaFormat)
|
||||
public val defaultEnvelopeFormats: List<EnvelopeFormatFactory> = listOf(
|
||||
TaggedEnvelopeFormat,
|
||||
TaglessEnvelopeFormat
|
||||
)
|
||||
|
||||
override val tag: PluginTag = PluginTag("io", group = PluginTag.DATAFORGE_GROUP)
|
||||
|
||||
override val type: KClass<out IOPlugin> = IOPlugin::class
|
||||
override fun invoke(meta: Meta, context: Context): IOPlugin = IOPlugin(meta)
|
||||
override fun build(context: Context, meta: Meta): IOPlugin = IOPlugin(meta)
|
||||
|
||||
public val WORK_DIRECTORY_KEY: Name = Name.of("io", "workDirectory")
|
||||
}
|
||||
}
|
||||
|
||||
@ThreadLocal
|
||||
internal val ioContext = Global.withEnv {
|
||||
name("IO")
|
||||
internal val ioContext = Context("IO") {
|
||||
plugin(IOPlugin)
|
||||
}
|
||||
|
||||
public val Context.io: IOPlugin get() = (if (this == Global) ioContext else this).fetch(IOPlugin)
|
||||
public val Context.io: IOPlugin
|
||||
get() = if (this == Global) {
|
||||
ioContext.request(IOPlugin)
|
||||
} else {
|
||||
request(IOPlugin)
|
||||
}
|
@ -1,59 +1,48 @@
|
||||
@file:Suppress("UNUSED_PARAMETER")
|
||||
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.readString
|
||||
import kotlinx.io.writeString
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.meta.node
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.toJson
|
||||
import space.kscience.dataforge.meta.toMetaItem
|
||||
import kotlin.reflect.KType
|
||||
import kotlin.reflect.typeOf
|
||||
import space.kscience.dataforge.meta.toMeta
|
||||
|
||||
/**
|
||||
* A Json format for Meta representation
|
||||
*/
|
||||
public class JsonMetaFormat(private val json: Json = DEFAULT_JSON) : MetaFormat {
|
||||
|
||||
override val type: KType get() = typeOf<Meta>()
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?) {
|
||||
val jsonObject = meta.toJson(descriptor)
|
||||
output.writeUtf8String(json.encodeToString(JsonObject.serializer(), jsonObject))
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?) {
|
||||
val jsonElement = meta.toJson(descriptor)
|
||||
sink.writeString(json.encodeToString(JsonElement.serializer(), jsonElement))
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
}
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta {
|
||||
val str = input.readUtf8String()//readByteArray().decodeToString()
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta {
|
||||
val str = source.readString()
|
||||
val jsonElement = json.parseToJsonElement(str)
|
||||
val item = jsonElement.toMetaItem(descriptor)
|
||||
return item.node ?: Meta.EMPTY
|
||||
return jsonElement.toMeta(descriptor)
|
||||
}
|
||||
|
||||
public companion object : MetaFormatFactory {
|
||||
public val DEFAULT_JSON: Json = Json { prettyPrint = true }
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): MetaFormat = default
|
||||
override fun build(context: Context, meta: Meta): MetaFormat = default
|
||||
|
||||
override val shortName: String = "json"
|
||||
override val key: Short = 0x4a53//"JS"
|
||||
|
||||
private val default = JsonMetaFormat()
|
||||
|
||||
override fun writeMeta(output: Output, meta: Meta, descriptor: NodeDescriptor?): Unit =
|
||||
default.run { writeMeta(output, meta, descriptor) }
|
||||
override fun writeMeta(sink: Sink, meta: Meta, descriptor: MetaDescriptor?): Unit =
|
||||
default.run { writeMeta(sink, meta, descriptor) }
|
||||
|
||||
override fun readMeta(input: Input, descriptor: NodeDescriptor?): Meta =
|
||||
default.run { readMeta(input, descriptor) }
|
||||
override fun readMeta(source: Source, descriptor: MetaDescriptor?): Meta =
|
||||
default.run { readMeta(source, descriptor) }
|
||||
}
|
||||
}
|
||||
|
@ -1,14 +1,15 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.Input
|
||||
import io.ktor.utils.io.core.Output
|
||||
import io.ktor.utils.io.core.use
|
||||
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.buffered
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.io.MetaFormatFactory.Companion.META_FORMAT_TYPE
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.misc.Type
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.misc.DfId
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.plus
|
||||
@ -19,24 +20,25 @@ import kotlin.reflect.typeOf
|
||||
* A format for meta serialization
|
||||
*/
|
||||
public interface MetaFormat : IOFormat<Meta> {
|
||||
|
||||
override val type: KType get() = typeOf<Meta>()
|
||||
|
||||
override fun writeObject(output: Output, obj: Meta) {
|
||||
writeMeta(output, obj, null)
|
||||
override fun writeTo(sink: Sink, obj: Meta) {
|
||||
writeMeta(sink, obj, null)
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Meta = readMeta(input)
|
||||
override fun readFrom(source: Source): Meta = readMeta(source)
|
||||
|
||||
public fun writeMeta(
|
||||
output: Output,
|
||||
sink: Sink,
|
||||
meta: Meta,
|
||||
descriptor: NodeDescriptor? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
)
|
||||
|
||||
public fun readMeta(input: Input, descriptor: NodeDescriptor? = null): Meta
|
||||
public fun readMeta(source: Source, descriptor: MetaDescriptor? = null): Meta
|
||||
}
|
||||
|
||||
@Type(META_FORMAT_TYPE)
|
||||
@DfId(META_FORMAT_TYPE)
|
||||
public interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
|
||||
public val shortName: String
|
||||
|
||||
@ -46,25 +48,23 @@ public interface MetaFormatFactory : IOFormatFactory<Meta>, MetaFormat {
|
||||
|
||||
public val key: Short get() = name.hashCode().toShort()
|
||||
|
||||
override operator fun invoke(meta: Meta, context: Context): MetaFormat
|
||||
override fun build(context: Context, meta: Meta): MetaFormat
|
||||
|
||||
public companion object {
|
||||
public const val META_FORMAT_TYPE: String = "io.format.meta"
|
||||
}
|
||||
}
|
||||
|
||||
public fun Meta.toString(format: MetaFormat): String = buildByteArray {
|
||||
public fun Meta.toString(format: MetaFormat): String = ByteArray {
|
||||
format.run {
|
||||
writeObject(this@buildByteArray, this@toString)
|
||||
writeTo(this@ByteArray, this@toString)
|
||||
}
|
||||
}.decodeToString()
|
||||
|
||||
public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory())
|
||||
public fun Meta.toString(formatFactory: MetaFormatFactory): String = toString(formatFactory.build(Global, Meta.EMPTY))
|
||||
|
||||
public fun MetaFormat.parse(str: String): Meta {
|
||||
return ByteReadPacket(str.encodeToByteArray()).use { readObject(it) }
|
||||
}
|
||||
public fun MetaFormat.parse(str: String): Meta = readFrom(StringSource(str).buffered())
|
||||
|
||||
public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = invoke(formatMeta).parse(str)
|
||||
public fun MetaFormatFactory.parse(str: String, formatMeta: Meta): Meta = build(Global, formatMeta).parse(str)
|
||||
|
||||
|
||||
|
@ -1,16 +1,16 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.enum
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.plus
|
||||
import space.kscience.dataforge.names.toName
|
||||
|
||||
|
||||
/**
|
||||
* A streaming-friendly envelope format with a short binary tag.
|
||||
@ -19,79 +19,79 @@ import space.kscience.dataforge.names.toName
|
||||
public class TaggedEnvelopeFormat(
|
||||
public val io: IOPlugin,
|
||||
public val version: VERSION = VERSION.DF02,
|
||||
public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat,
|
||||
) : EnvelopeFormat {
|
||||
|
||||
// private val metaFormat = io.metaFormat(metaFormatKey)
|
||||
// ?: error("Meta format with key $metaFormatKey could not be resolved in $io")
|
||||
|
||||
|
||||
private fun Tag.toBinary() = Binary(24) {
|
||||
writeRawString(START_SEQUENCE)
|
||||
writeRawString(version.name)
|
||||
private fun Tag.toBinary() = Binary {
|
||||
write(START_SEQUENCE)
|
||||
writeString(version.name)
|
||||
writeShort(metaFormatKey)
|
||||
writeUInt(metaSize)
|
||||
when (version) {
|
||||
VERSION.DF02 -> {
|
||||
writeUInt(dataSize.toUInt())
|
||||
}
|
||||
|
||||
VERSION.DF03 -> {
|
||||
writeULong(dataSize)
|
||||
}
|
||||
}
|
||||
writeRawString(END_SEQUENCE)
|
||||
write(END_SEQUENCE)
|
||||
}
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory.invoke(formatMeta, this@TaggedEnvelopeFormat.io.context)
|
||||
val metaBytes = metaFormat.toBinary(envelope.meta)
|
||||
val actualSize: ULong = (envelope.data?.size ?: 0).toULong()
|
||||
val metaFormat = metaFormatFactory.build(io.context, Meta.EMPTY)
|
||||
val metaBytes = Binary(obj.meta, metaFormat)
|
||||
val actualSize: ULong = (obj.data?.size ?: 0).toULong()
|
||||
val tag = Tag(metaFormatFactory.key, metaBytes.size.toUInt() + 2u, actualSize)
|
||||
output.writeBinary(tag.toBinary())
|
||||
output.writeBinary(metaBytes)
|
||||
output.writeRawString("\r\n")
|
||||
envelope.data?.let {
|
||||
output.writeBinary(it)
|
||||
sink.writeBinary(tag.toBinary())
|
||||
sink.writeBinary(metaBytes)
|
||||
sink.writeString("\r\n")
|
||||
obj.data?.let {
|
||||
sink.writeBinary(it)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an envelope from input into memory
|
||||
*
|
||||
* @param input an input to read from
|
||||
* @param source an input to read from
|
||||
* @param formats a collection of meta formats to resolve
|
||||
*/
|
||||
override fun readObject(input: Input): Envelope {
|
||||
val tag = input.readTag(this.version)
|
||||
override fun readFrom(source: Source): Envelope {
|
||||
val tag = source.readTag(this.version)
|
||||
|
||||
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
|
||||
?: error("Meta format with key ${tag.metaFormatKey} not found")
|
||||
|
||||
val metaBinary = input.readBinary(tag.metaSize.toInt())
|
||||
val metaBinary = source.readBinary(tag.metaSize.toInt())
|
||||
|
||||
val meta: Meta = metaFormat.readObject(metaBinary)
|
||||
val meta: Meta = metaFormat.readFrom(metaBinary)
|
||||
|
||||
val data = input.readBinary(tag.dataSize.toInt())
|
||||
val data = source.readBinary(tag.dataSize.toInt())
|
||||
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope {
|
||||
val tag = input.readTag(this.version)
|
||||
override fun readFrom(binary: Binary): Envelope = binary.read {
|
||||
val tag = readTag(version)
|
||||
|
||||
val metaFormat = io.resolveMetaFormat(tag.metaFormatKey)
|
||||
?: error("Meta format with key ${tag.metaFormatKey} not found")
|
||||
|
||||
val metaBinary = input.readBinary(tag.metaSize.toInt())
|
||||
val metaBinary = readBinary(tag.metaSize.toInt())
|
||||
|
||||
val meta: Meta = metaFormat.readObject(metaBinary)
|
||||
val meta: Meta = metaFormat.readFrom(metaBinary)
|
||||
|
||||
|
||||
return PartialEnvelope(meta, version.tagSize + tag.metaSize, tag.dataSize)
|
||||
SimpleEnvelope(meta, binary.view((version.tagSize + tag.metaSize).toInt(), tag.dataSize.toInt()))
|
||||
}
|
||||
|
||||
private data class Tag(
|
||||
@ -105,23 +105,16 @@ public class TaggedEnvelopeFormat(
|
||||
DF03(24u)
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
META_KEY put {
|
||||
"version" put version
|
||||
}
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
private const val START_SEQUENCE = "#~"
|
||||
private const val END_SEQUENCE = "~#\r\n"
|
||||
private val START_SEQUENCE = "#~".toAsciiByteString()
|
||||
private val END_SEQUENCE = "~#\r\n".toAsciiByteString()
|
||||
|
||||
override val name: Name = super.name + "tagged"
|
||||
override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + "tagged"
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat {
|
||||
val io = context.io
|
||||
|
||||
val metaFormatName = meta["name"].string?.toName() ?: JsonMetaFormat.name
|
||||
val metaFormatName = meta["name"].string?.let { Name.parse(it) } ?: JsonMetaFormat.name
|
||||
//Check if appropriate factory exists
|
||||
io.metaFormatFactories.find { it.name == metaFormatName } ?: error("Meta format could not be resolved")
|
||||
|
||||
@ -130,57 +123,48 @@ public class TaggedEnvelopeFormat(
|
||||
return TaggedEnvelopeFormat(io, version)
|
||||
}
|
||||
|
||||
private fun Input.readTag(version: VERSION): Tag {
|
||||
val start = readRawString(2)
|
||||
private fun Source.readTag(version: VERSION): Tag {
|
||||
val start = readByteString(2)
|
||||
if (start != START_SEQUENCE) error("The input is not an envelope")
|
||||
val versionString = readRawString(4)
|
||||
if (version.name != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
|
||||
val versionString = readByteString(4)
|
||||
if (version.name.toAsciiByteString() != versionString) error("Wrong version of DataForge: expected $version but found $versionString")
|
||||
val metaFormatKey = readShort()
|
||||
val metaLength = readUInt()
|
||||
val dataLength: ULong = when (version) {
|
||||
VERSION.DF02 -> readUInt().toULong()
|
||||
VERSION.DF03 -> readULong()
|
||||
}
|
||||
val end = readRawString(4)
|
||||
val end = readByteString(4)
|
||||
if (end != END_SEQUENCE) error("The input is not an envelope")
|
||||
return Tag(metaFormatKey, metaLength, dataLength)
|
||||
}
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
|
||||
return try {
|
||||
binary.read{
|
||||
val header = readRawString(6)
|
||||
return@read when (header.substring(2..5)) {
|
||||
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
|
||||
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
|
||||
else -> null
|
||||
}
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? = try {
|
||||
binary.read {
|
||||
val header = readByteString(6)
|
||||
when (header.substring(2, 6).decodeToString()) {
|
||||
VERSION.DF02.name -> TaggedEnvelopeFormat(io, VERSION.DF02)
|
||||
VERSION.DF03.name -> TaggedEnvelopeFormat(io, VERSION.DF03)
|
||||
else -> null
|
||||
}
|
||||
} catch (ex: Exception) {
|
||||
null
|
||||
}
|
||||
} catch (ex: Exception) {
|
||||
null
|
||||
}
|
||||
|
||||
private val default by lazy { invoke(context = ioContext) }
|
||||
private val default by lazy { build(Global, Meta.EMPTY) }
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope =
|
||||
default.run { readPartial(input) }
|
||||
override fun readFrom(binary: Binary): Envelope =
|
||||
default.run { readFrom(binary) }
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
): Unit = default.run {
|
||||
writeEnvelope(
|
||||
output,
|
||||
envelope,
|
||||
metaFormatFactory,
|
||||
formatMeta
|
||||
)
|
||||
writeTo(sink, obj)
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope = default.readObject(input)
|
||||
override fun readFrom(source: Source): Envelope = default.readFrom(source)
|
||||
}
|
||||
|
||||
}
|
@ -1,175 +1,104 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import space.kscience.dataforge.context.Context
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.META_KEY
|
||||
import space.kscience.dataforge.io.IOFormat.Companion.NAME_KEY
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.isEmpty
|
||||
import space.kscience.dataforge.meta.string
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.asName
|
||||
import kotlin.collections.set
|
||||
import space.kscience.dataforge.names.plus
|
||||
|
||||
/**
|
||||
* A text envelope format with human-readable tag.
|
||||
* A text envelope format based on block separators.
|
||||
* TODO add description
|
||||
*/
|
||||
public class TaglessEnvelopeFormat(
|
||||
public val io: IOPlugin,
|
||||
public val meta: Meta = Meta.EMPTY,
|
||||
public val metaFormatFactory: MetaFormatFactory = JsonMetaFormat,
|
||||
) : EnvelopeFormat {
|
||||
|
||||
private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
|
||||
private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START
|
||||
// private val metaStart = meta[META_START_PROPERTY].string ?: DEFAULT_META_START
|
||||
// private val dataStart = meta[DATA_START_PROPERTY].string ?: DEFAULT_DATA_START
|
||||
|
||||
private fun Output.writeProperty(key: String, value: Any) {
|
||||
writeFully("#? $key: $value;\r\n".encodeToByteArray())
|
||||
}
|
||||
// private fun Output.writeProperty(key: String, value: Any) {
|
||||
// writeFully("#? $key: $value;\r\n".encodeToByteArray())
|
||||
// }
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
) {
|
||||
val metaFormat = metaFormatFactory(formatMeta, this.io.context)
|
||||
val metaFormat = metaFormatFactory.build(this.io.context, meta)
|
||||
|
||||
//printing header
|
||||
output.writeRawString(TAGLESS_ENVELOPE_HEADER + "\r\n")
|
||||
|
||||
//printing all properties
|
||||
output.writeProperty(META_TYPE_PROPERTY,
|
||||
metaFormatFactory.shortName)
|
||||
//TODO add optional metaFormat properties
|
||||
val actualSize: Int = envelope.data?.size ?: 0
|
||||
|
||||
output.writeProperty(DATA_LENGTH_PROPERTY, actualSize)
|
||||
sink.write(TAGLESS_ENVELOPE_HEADER)
|
||||
sink.writeString("\r\n")
|
||||
|
||||
//Printing meta
|
||||
if (!envelope.meta.isEmpty()) {
|
||||
val metaBytes = metaFormat.toBinary(envelope.meta)
|
||||
output.writeProperty(META_LENGTH_PROPERTY,
|
||||
metaBytes.size + 2)
|
||||
output.writeUtf8String(this.metaStart + "\r\n")
|
||||
output.writeBinary(metaBytes)
|
||||
output.writeRawString("\r\n")
|
||||
if (!obj.meta.isEmpty()) {
|
||||
val metaBinary = Binary(obj.meta, metaFormat)
|
||||
sink.writeString(META_START + "-${metaFormatFactory.shortName}\r\n")
|
||||
sink.writeBinary(metaBinary)
|
||||
sink.writeString("\r\n")
|
||||
}
|
||||
|
||||
//Printing data
|
||||
envelope.data?.let { data ->
|
||||
output.writeUtf8String(this.dataStart + "\r\n")
|
||||
output.writeBinary(data)
|
||||
obj.data?.let { data ->
|
||||
//val actualSize: Int = envelope.data?.size ?: 0
|
||||
sink.writeString(DATA_START + "\r\n")
|
||||
sink.writeBinary(data)
|
||||
}
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope {
|
||||
var line: String
|
||||
do {
|
||||
line = input.readSafeUtf8Line() // ?: error("Input does not contain tagless envelope header")
|
||||
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
|
||||
val properties = HashMap<String, String>()
|
||||
|
||||
line = ""
|
||||
while (line.isBlank() || line.startsWith("#?")) {
|
||||
if (line.startsWith("#?")) {
|
||||
val match = propertyPattern.find(line)
|
||||
?: error("Line $line does not match property declaration pattern")
|
||||
val (key, value) = match.destructured
|
||||
properties[key] = value
|
||||
}
|
||||
//If can't read line, return envelope without data
|
||||
if (input.endOfInput) return SimpleEnvelope(Meta.EMPTY, null)
|
||||
line = input.readSafeUtf8Line()
|
||||
}
|
||||
override fun readFrom(source: Source): Envelope {
|
||||
//read preamble
|
||||
source.discardWithSeparator(
|
||||
TAGLESS_ENVELOPE_HEADER,
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
var meta: Meta = Meta.EMPTY
|
||||
|
||||
if (line.startsWith(metaStart)) {
|
||||
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
|
||||
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
|
||||
meta = if (metaSize != null) {
|
||||
metaFormat.readObject(input.readBinary(metaSize))
|
||||
} else {
|
||||
metaFormat.readObject(input)
|
||||
var data: Binary? = null
|
||||
|
||||
source.discardWithSeparator(
|
||||
SEPARATOR_PREFIX,
|
||||
atMost = 1024,
|
||||
)
|
||||
|
||||
var header: String = ByteArray {
|
||||
source.readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
}.decodeToString()
|
||||
|
||||
while (!source.exhausted()) {
|
||||
val block = ByteArray {
|
||||
source.readWithSeparatorTo(this, SEPARATOR_PREFIX)
|
||||
}
|
||||
|
||||
val nextHeader = ByteArray {
|
||||
source.readWithSeparatorTo(this, "\n".encodeToByteString())
|
||||
}.decodeToString()
|
||||
|
||||
//terminate on end
|
||||
if (header.startsWith("END")) break
|
||||
|
||||
|
||||
if (header.startsWith("META")) {
|
||||
//TODO check format
|
||||
val metaFormat: MetaFormatFactory = JsonMetaFormat
|
||||
meta = metaFormat.readMeta(ByteArraySource(block).buffered())
|
||||
}
|
||||
|
||||
if (header.startsWith("DATA")) {
|
||||
data = block.asBinary()
|
||||
}
|
||||
header = nextHeader
|
||||
}
|
||||
|
||||
do {
|
||||
try {
|
||||
line = input.readSafeUtf8Line()
|
||||
} catch (ex: EOFException) {
|
||||
//returning an Envelope without data if end of input is reached
|
||||
return SimpleEnvelope(meta, null)
|
||||
}
|
||||
} while (!line.startsWith(dataStart))
|
||||
|
||||
val data: Binary = if (properties.containsKey(DATA_LENGTH_PROPERTY)) {
|
||||
input.readBinary(properties[DATA_LENGTH_PROPERTY]!!.toInt())
|
||||
// val bytes = ByteArray(properties[DATA_LENGTH_PROPERTY]!!.toInt())
|
||||
// readByteArray(bytes)
|
||||
// bytes.asBinary()
|
||||
} else {
|
||||
Binary {
|
||||
input.copyTo(this)
|
||||
}
|
||||
}
|
||||
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope {
|
||||
var offset = 0u
|
||||
var line: String
|
||||
do {
|
||||
line = input.readSafeUtf8Line()// ?: error("Input does not contain tagless envelope header")
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} while (!line.startsWith(TAGLESS_ENVELOPE_HEADER))
|
||||
val properties = HashMap<String, String>()
|
||||
|
||||
line = ""
|
||||
while (line.isBlank() || line.startsWith("#?")) {
|
||||
if (line.startsWith("#?")) {
|
||||
val match = propertyPattern.find(line)
|
||||
?: error("Line $line does not match property declaration pattern")
|
||||
val (key, value) = match.destructured
|
||||
properties[key] = value
|
||||
}
|
||||
try {
|
||||
line = input.readSafeUtf8Line()
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
} catch (ex: EOFException) {
|
||||
return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
|
||||
}
|
||||
}
|
||||
|
||||
var meta: Meta = Meta.EMPTY
|
||||
|
||||
if (line.startsWith(metaStart)) {
|
||||
val metaFormat = properties[META_TYPE_PROPERTY]?.let { io.resolveMetaFormat(it) } ?: JsonMetaFormat
|
||||
val metaSize = properties[META_LENGTH_PROPERTY]?.toInt()
|
||||
meta = if (metaSize != null) {
|
||||
offset += metaSize.toUInt()
|
||||
metaFormat.readObject(input.readBinary(metaSize))
|
||||
} else {
|
||||
error("Can't partially read an envelope with undefined meta size")
|
||||
}
|
||||
}
|
||||
|
||||
do {
|
||||
line = input.readSafeUtf8Line() //?: return PartialEnvelope(Meta.EMPTY, offset.toUInt(), 0.toULong())
|
||||
offset += line.encodeToByteArray().size.toUInt()
|
||||
//returning an Envelope without data if end of input is reached
|
||||
} while (!line.startsWith(dataStart))
|
||||
|
||||
val dataSize = properties[DATA_LENGTH_PROPERTY]?.toULong()
|
||||
return PartialEnvelope(meta, offset, dataSize)
|
||||
}
|
||||
|
||||
override fun toMeta(): Meta = Meta {
|
||||
NAME_KEY put name.toString()
|
||||
META_KEY put meta
|
||||
return Envelope(meta, data)
|
||||
}
|
||||
|
||||
public companion object : EnvelopeFormatFactory {
|
||||
@ -183,45 +112,41 @@ public class TaglessEnvelopeFormat(
|
||||
|
||||
public const val TAGLESS_ENVELOPE_TYPE: String = "tagless"
|
||||
|
||||
public const val TAGLESS_ENVELOPE_HEADER: String = "#~DFTL~#"
|
||||
public const val META_START_PROPERTY: String = "metaSeparator"
|
||||
public const val DEFAULT_META_START: String = "#~META~#"
|
||||
public const val DATA_START_PROPERTY: String = "dataSeparator"
|
||||
public const val DEFAULT_DATA_START: String = "#~DATA~#"
|
||||
public val SEPARATOR_PREFIX: ByteString = "\n#~".encodeToByteString()
|
||||
|
||||
public val TAGLESS_ENVELOPE_HEADER: ByteString = "#~DFTL".encodeToByteString()
|
||||
|
||||
// public const val META_START_PROPERTY: String = "metaSeparator"
|
||||
public const val META_START: String = "#~META"
|
||||
|
||||
// public const val DATA_START_PROPERTY: String = "dataSeparator"
|
||||
public const val DATA_START: String = "#~DATA"
|
||||
|
||||
public const val END: String = "#~END"
|
||||
|
||||
public const val code: Int = 0x4446544c //DFTL
|
||||
|
||||
override val name: Name = TAGLESS_ENVELOPE_TYPE.asName()
|
||||
override val name: Name = EnvelopeFormatFactory.ENVELOPE_FACTORY_NAME + TAGLESS_ENVELOPE_TYPE
|
||||
|
||||
override fun invoke(meta: Meta, context: Context): EnvelopeFormat {
|
||||
return TaglessEnvelopeFormat(context.io, meta)
|
||||
}
|
||||
override fun build(context: Context, meta: Meta): EnvelopeFormat = TaglessEnvelopeFormat(context.io, meta)
|
||||
|
||||
private val default by lazy { invoke(context = ioContext) }
|
||||
private val default by lazy { build(Global, Meta.EMPTY) }
|
||||
|
||||
override fun readPartial(input: Input): PartialEnvelope =
|
||||
default.run { readPartial(input) }
|
||||
override fun readFrom(binary: Binary): Envelope = default.run { readFrom(binary) }
|
||||
|
||||
override fun writeEnvelope(
|
||||
output: Output,
|
||||
envelope: Envelope,
|
||||
metaFormatFactory: MetaFormatFactory,
|
||||
formatMeta: Meta,
|
||||
override fun writeTo(
|
||||
sink: Sink,
|
||||
obj: Envelope,
|
||||
): Unit = default.run {
|
||||
writeEnvelope(
|
||||
output,
|
||||
envelope,
|
||||
metaFormatFactory,
|
||||
formatMeta
|
||||
)
|
||||
writeTo(sink, obj)
|
||||
}
|
||||
|
||||
override fun readObject(input: Input): Envelope = default.readObject(input)
|
||||
override fun readFrom(source: Source): Envelope = default.readFrom(source)
|
||||
|
||||
override fun peekFormat(io: IOPlugin, binary: Binary): EnvelopeFormat? {
|
||||
return try {
|
||||
binary.read {
|
||||
val string = readRawString(TAGLESS_ENVELOPE_HEADER.length)
|
||||
val string = readByteString(TAGLESS_ENVELOPE_HEADER.size)
|
||||
return@read if (string == TAGLESS_ENVELOPE_HEADER) {
|
||||
TaglessEnvelopeFormat(io)
|
||||
} else {
|
||||
|
@ -1,51 +1,235 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.charsets.Charsets
|
||||
import io.ktor.utils.io.charsets.decodeExactBytes
|
||||
import io.ktor.utils.io.core.*
|
||||
import kotlinx.io.*
|
||||
import kotlinx.io.bytestring.ByteString
|
||||
import kotlinx.io.bytestring.decodeToString
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import kotlin.math.min
|
||||
|
||||
public fun Output.writeRawString(str: String) {
|
||||
writeFully(str.toByteArray(Charsets.ISO_8859_1))
|
||||
/**
|
||||
* Convert a string literal, containing only ASCII characters to a [ByteString].
|
||||
* Throws an error if there are non-ASCII characters.
|
||||
*/
|
||||
public fun String.toAsciiByteString(): ByteString {
|
||||
val bytes = ByteArray(length) {
|
||||
val char = get(it)
|
||||
val code = char.code
|
||||
if (code > Byte.MAX_VALUE) error("Symbol $char is not ASCII symbol") else code.toByte()
|
||||
}
|
||||
return ByteString(bytes)
|
||||
}
|
||||
|
||||
public fun Output.writeUtf8String(str: String) {
|
||||
writeFully(str.encodeToByteArray())
|
||||
}
|
||||
public inline fun Buffer(block: Sink.() -> Unit): Buffer = Buffer().apply(block)
|
||||
|
||||
@OptIn(ExperimentalIoApi::class)
|
||||
public fun Input.readRawString(size: Int): String {
|
||||
return Charsets.ISO_8859_1.newDecoder().decodeExactBytes(this, size)
|
||||
}
|
||||
//public fun Source.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found")
|
||||
|
||||
public fun Input.readUtf8String(): String = readBytes().decodeToString()
|
||||
public inline fun ByteArray(block: Sink.() -> Unit): ByteArray =
|
||||
Buffer(block).readByteArray()
|
||||
|
||||
public fun Input.readSafeUtf8Line(): String = readUTF8Line() ?: error("Line not found")
|
||||
public inline fun Binary(block: Sink.() -> Unit): Binary =
|
||||
ByteArray(block).asBinary()
|
||||
|
||||
public inline fun buildByteArray(expectedSize: Int = 16, block: Output.() -> Unit): ByteArray {
|
||||
val builder = BytePacketBuilder(expectedSize)
|
||||
builder.block()
|
||||
return builder.build().readBytes()
|
||||
}
|
||||
|
||||
public inline fun Binary(expectedSize: Int = 16, block: Output.() -> Unit): Binary =
|
||||
buildByteArray(expectedSize, block).asBinary()
|
||||
public operator fun Binary.get(range: IntRange): Binary = view(range.first, range.last - range.first)
|
||||
|
||||
/**
|
||||
* View section of a [Binary] as an independent binary
|
||||
* Return inferred [EnvelopeFormat] if only one format could read given file. If no format accepts the binary, return null. If
|
||||
* multiple formats accept binary, throw an error.
|
||||
*/
|
||||
public class BinaryView(private val source: Binary, private val start: Int, override val size: Int) : Binary {
|
||||
|
||||
init {
|
||||
require(start > 0)
|
||||
require(start + size <= source.size) { "View boundary is outside source binary size" }
|
||||
public fun IOPlugin.peekBinaryEnvelopeFormat(binary: Binary): EnvelopeFormat? {
|
||||
val formats = envelopeFormatFactories.mapNotNull { factory ->
|
||||
factory.peekFormat(this@peekBinaryEnvelopeFormat, binary)
|
||||
}
|
||||
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
|
||||
return source.read(start + offset, min(size, atMost), block)
|
||||
return when (formats.size) {
|
||||
0 -> null
|
||||
1 -> formats.first()
|
||||
else -> error("Envelope format binary recognition clash: $formats")
|
||||
}
|
||||
}
|
||||
|
||||
public fun Binary.view(start: Int, size: Int): BinaryView = BinaryView(this, start, size)
|
||||
/**
|
||||
* A zero-copy read from
|
||||
*/
|
||||
@DFExperimental
|
||||
public fun IOPlugin.readEnvelope(
|
||||
binary: Binary,
|
||||
readNonEnvelopes: Boolean = false,
|
||||
formatPicker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryEnvelopeFormat,
|
||||
): Envelope = formatPicker(binary)?.readFrom(binary) ?: if (readNonEnvelopes) {
|
||||
// if no format accepts file, read it as binary
|
||||
Envelope(Meta.EMPTY, binary)
|
||||
} else error("Can't infer format for $binary")
|
||||
|
||||
public operator fun Binary.get(range: IntRange): BinaryView = view(range.first, range.last - range.first)
|
||||
@DFExperimental
|
||||
public fun IOPlugin.readEnvelope(
|
||||
string: String,
|
||||
readNonEnvelopes: Boolean = false,
|
||||
formatPicker: IOPlugin.(Binary) -> EnvelopeFormat? = IOPlugin::peekBinaryEnvelopeFormat,
|
||||
): Envelope = readEnvelope(string.encodeToByteArray().asBinary(), readNonEnvelopes, formatPicker)
|
||||
|
||||
|
||||
private class RingByteArray(
|
||||
private val buffer: ByteArray,
|
||||
private var startIndex: Int = 0,
|
||||
var size: Int = 0,
|
||||
) {
|
||||
operator fun get(index: Int): Byte {
|
||||
require(index >= 0) { "Index must be positive" }
|
||||
require(index < size) { "Index $index is out of circular buffer size $size" }
|
||||
return buffer[startIndex.forward(index)]
|
||||
}
|
||||
|
||||
fun isFull(): Boolean = size == buffer.size
|
||||
|
||||
fun push(element: Byte) {
|
||||
buffer[startIndex.forward(size)] = element
|
||||
if (isFull()) startIndex++ else size++
|
||||
|
||||
}
|
||||
|
||||
private fun Int.forward(n: Int): Int = (this + n) % (buffer.size)
|
||||
|
||||
fun contentEquals(inputArray: ByteArray): Boolean = when {
|
||||
inputArray.size != buffer.size -> false
|
||||
size < buffer.size -> false
|
||||
else -> inputArray.indices.all { inputArray[it] == get(it) }
|
||||
}
|
||||
|
||||
fun contentEquals(byteString: ByteString): Boolean = when {
|
||||
byteString.size != buffer.size -> false
|
||||
size < buffer.size -> false
|
||||
else -> (0 until byteString.size).all { byteString[it] == get(it) }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private fun RingByteArray.toArray(): ByteArray = ByteArray(size) { get(it) }
|
||||
|
||||
/**
|
||||
* Read [Source] into [output] until designated multibyte [separator] and optionally continues until
|
||||
* the end of the line after it. Throw error if [separator] not found and [atMost] bytes are read.
|
||||
* Also fails if [separator] not found until the end of input.
|
||||
*
|
||||
* The Separator itself is not read into [Sink].
|
||||
*
|
||||
* @param errorOnEof if true error is thrown if separator is never encountered
|
||||
*
|
||||
* @return bytes actually being read, including separator
|
||||
*/
|
||||
public fun Source.readWithSeparatorTo(
|
||||
output: Sink?,
|
||||
separator: ByteString,
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int {
|
||||
var counter = 0
|
||||
val rb = RingByteArray(ByteArray(separator.size))
|
||||
|
||||
while (!exhausted()) {
|
||||
val byte = readByte()
|
||||
counter++
|
||||
if (counter >= atMost) error("Maximum number of bytes to be read $atMost reached.")
|
||||
rb.push(byte)
|
||||
if (rb.contentEquals(separator)) {
|
||||
return counter
|
||||
} else if (rb.isFull()) {
|
||||
output?.writeByte(rb[0])
|
||||
}
|
||||
}
|
||||
|
||||
if (errorOnEof) {
|
||||
error("Read to the end of input without encountering ${separator.decodeToString()}")
|
||||
} else {
|
||||
for (i in 1 until rb.size) {
|
||||
output?.writeByte(rb[i])
|
||||
}
|
||||
counter += (rb.size - 1)
|
||||
return counter
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discard all bytes until [separator] is encountered. Separator is discarded sa well.
|
||||
* Return the total number of bytes read.
|
||||
*/
|
||||
public fun Source.discardWithSeparator(
|
||||
separator: ByteString,
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int = readWithSeparatorTo(null, separator, atMost, errorOnEof)
|
||||
|
||||
/**
|
||||
* Discard all symbol until newline is discovered. Carriage return is not discarded.
|
||||
*/
|
||||
public fun Source.discardLine(
|
||||
atMost: Int = Int.MAX_VALUE,
|
||||
errorOnEof: Boolean = false,
|
||||
): Int = discardWithSeparator("\n".encodeToByteString(), atMost, errorOnEof)
|
||||
|
||||
|
||||
/**
|
||||
* A [Source] based on [ByteArray]
|
||||
*/
|
||||
internal class ByteArraySource(
|
||||
private val byteArray: ByteArray,
|
||||
private val offset: Int = 0,
|
||||
private val size: Int = byteArray.size - offset,
|
||||
) : RawSource {
|
||||
|
||||
init {
|
||||
require(offset >= 0) { "Offset must be positive" }
|
||||
require(offset + size <= byteArray.size) { "End index is ${offset + size}, but the array size is ${byteArray.size}" }
|
||||
}
|
||||
|
||||
private var pointer = offset
|
||||
|
||||
override fun close() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
override fun readAtMostTo(sink: Buffer, byteCount: Long): Long {
|
||||
if (pointer == offset + size) return -1
|
||||
val byteRead = min(byteCount.toInt(), (size + offset - pointer))
|
||||
sink.write(byteArray, pointer, pointer + byteRead)
|
||||
pointer += byteRead
|
||||
return byteRead.toLong()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A [Source] based on [String]
|
||||
*/
|
||||
public class StringSource(
|
||||
public val string: String,
|
||||
public val offset: Int = 0,
|
||||
public val size: Int = string.length - offset,
|
||||
) : RawSource {
|
||||
|
||||
private var pointer = offset
|
||||
|
||||
override fun close() {
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
override fun readAtMostTo(sink: Buffer, byteCount: Long): Long {
|
||||
if (pointer == offset + size) return -1
|
||||
val byteRead = min(byteCount.toInt(), (size + offset - pointer))
|
||||
sink.writeString(string, pointer, pointer + byteRead)
|
||||
pointer += byteRead
|
||||
return byteRead.toLong()
|
||||
}
|
||||
}
|
||||
|
||||
public fun Sink.writeDouble(value: Double) {
|
||||
writeLong(value.toBits())
|
||||
}
|
||||
|
||||
public fun Source.readDouble(): Double = Double.fromBits(readLong())
|
||||
|
||||
public fun Sink.writeFloat(value: Float) {
|
||||
writeInt(value.toBits())
|
||||
}
|
||||
|
||||
public fun Source.readFloat(): Float = Float.fromBits(readInt())
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.readInt
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.readDouble
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlinx.io.writeString
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -9,42 +9,56 @@ import kotlin.test.assertEquals
|
||||
class EnvelopeFormatTest {
|
||||
val envelope = Envelope {
|
||||
type = "test.format"
|
||||
meta{
|
||||
meta {
|
||||
"d" put 22.2
|
||||
}
|
||||
data{
|
||||
writeDouble(22.2)
|
||||
// repeat(2000){
|
||||
// writeInt(it)
|
||||
// }
|
||||
data {
|
||||
writeString("12345678")
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testTaggedFormat(){
|
||||
TaggedEnvelopeFormat.run {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
//println(byteArray.decodeToString())
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta,res.meta)
|
||||
val double = res.data?.read {
|
||||
readDouble()
|
||||
}
|
||||
assertEquals(22.2, double)
|
||||
fun testTaggedFormat() = with(TaggedEnvelopeFormat) {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testTaglessFormat(){
|
||||
TaglessEnvelopeFormat.run {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
//println(byteArray.decodeToString())
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta,res.meta)
|
||||
val double = res.data?.read {
|
||||
readDouble()
|
||||
}
|
||||
assertEquals(22.2, double)
|
||||
fun testTaglessFormat() = with(TaglessEnvelopeFormat) {
|
||||
val byteArray = writeToByteArray(envelope)
|
||||
println(byteArray.decodeToString())
|
||||
val res = readFromByteArray(byteArray)
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testManualDftl() {
|
||||
val envelopeString = """
|
||||
#~DFTL
|
||||
#~META
|
||||
{
|
||||
"@envelope": {
|
||||
"type": "test.format"
|
||||
},
|
||||
"d": 22.2
|
||||
}
|
||||
#~DATA
|
||||
12345678
|
||||
""".trimIndent()
|
||||
val res = TaglessEnvelopeFormat.readFromByteArray(envelopeString.encodeToByteArray())
|
||||
assertEquals(envelope.meta, res.meta)
|
||||
val bytes = res.data?.read {
|
||||
readByteArray()
|
||||
}
|
||||
assertEquals("12345678", bytes?.decodeToString())
|
||||
}
|
||||
}
|
@ -1,17 +1,58 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.readBytes
|
||||
import kotlinx.io.buffered
|
||||
import kotlinx.io.bytestring.encodeToByteString
|
||||
import kotlinx.io.readByteArray
|
||||
import kotlinx.io.readLine
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFails
|
||||
|
||||
class IOTest {
|
||||
@Test
|
||||
fun readBytes() {
|
||||
val bytes = ByteArray(8) { it.toByte() }
|
||||
val input = ByteReadPacket(bytes)
|
||||
@Suppress("UNUSED_VARIABLE") val first = input.readBytes(4)
|
||||
val second = input.readBytes(4)
|
||||
val input = ByteArraySource(bytes).buffered()
|
||||
@Suppress("UNUSED_VARIABLE") val first = input.readByteArray(4)
|
||||
val second = input.readByteArray(4)
|
||||
assertEquals(4.toByte(), second[0])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun readUntilSeparator() {
|
||||
val source = """
|
||||
aaa
|
||||
bbb
|
||||
---
|
||||
ccc
|
||||
ddd
|
||||
""".trimIndent()
|
||||
|
||||
val binary = source.encodeToByteArray().asBinary()
|
||||
|
||||
binary.read {
|
||||
val array = ByteArray {
|
||||
val read = readWithSeparatorTo(this, "---".encodeToByteString()) + discardLine()
|
||||
assertEquals(12, read)
|
||||
}
|
||||
assertEquals("""
|
||||
aaa
|
||||
bbb
|
||||
""".trimIndent(),array.decodeToString().trim())
|
||||
assertEquals("ccc", readLine()?.trim())
|
||||
}
|
||||
|
||||
assertFails {
|
||||
binary.read {
|
||||
discardWithSeparator("---".encodeToByteString(), atMost = 3 )
|
||||
}
|
||||
}
|
||||
|
||||
assertFails {
|
||||
binary.read{
|
||||
discardWithSeparator("-+-".encodeToByteString(), errorOnEof = true)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
@ -2,35 +2,19 @@ package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.serialization.json.*
|
||||
import space.kscience.dataforge.meta.*
|
||||
import space.kscience.dataforge.meta.JsonMeta.Companion.JSON_ARRAY_KEY
|
||||
import space.kscience.dataforge.values.ListValue
|
||||
import space.kscience.dataforge.values.number
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = buildByteArray {
|
||||
format.writeObject(this@buildByteArray, this@toByteArray)
|
||||
|
||||
fun Meta.toByteArray(format: MetaFormat = JsonMetaFormat) = ByteArray {
|
||||
format.writeTo(this@ByteArray, this@toByteArray)
|
||||
}
|
||||
|
||||
fun MetaFormat.fromByteArray(packet: ByteArray): Meta {
|
||||
return packet.asBinary().read { readObject(this) }
|
||||
return packet.asBinary().read { readFrom(this) }
|
||||
}
|
||||
|
||||
class MetaFormatTest {
|
||||
@Test
|
||||
fun testBinaryMetaFormat() {
|
||||
val meta = Meta {
|
||||
"a" put 22
|
||||
"node" put {
|
||||
"b" put "DDD"
|
||||
"c" put 11.1
|
||||
"array" put doubleArrayOf(1.0, 2.0, 3.0)
|
||||
}
|
||||
}
|
||||
val bytes = meta.toByteArray(BinaryMetaFormat)
|
||||
val result = BinaryMetaFormat.fromByteArray(bytes)
|
||||
assertEquals(meta, result)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testJsonMetaFormat() {
|
||||
@ -51,36 +35,36 @@ class MetaFormatTest {
|
||||
if (meta[it] != result[it]) error("${meta[it]} != ${result[it]}")
|
||||
}
|
||||
|
||||
assertEquals<Meta>(meta, result)
|
||||
assertEquals(meta, result)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testJsonToMeta() {
|
||||
val json = buildJsonArray {
|
||||
//top level array
|
||||
add(buildJsonArray {
|
||||
add(JsonPrimitive(88))
|
||||
add(buildJsonObject {
|
||||
addJsonArray {
|
||||
add(88)
|
||||
addJsonObject {
|
||||
put("c", "aasdad")
|
||||
put("d", true)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
add("value")
|
||||
add(buildJsonArray {
|
||||
add(JsonPrimitive(1.0))
|
||||
add(JsonPrimitive(2.0))
|
||||
add(JsonPrimitive(3.0))
|
||||
})
|
||||
addJsonArray {
|
||||
add(1.0)
|
||||
add(2.0)
|
||||
add(3.0)
|
||||
}
|
||||
}
|
||||
val meta = json.toMetaItem().node!!
|
||||
val meta = json.toMeta()
|
||||
|
||||
assertEquals(true, meta["$JSON_ARRAY_KEY[0].$JSON_ARRAY_KEY[1].d"].boolean)
|
||||
assertEquals("value", meta["$JSON_ARRAY_KEY[1]"].string)
|
||||
assertEquals(listOf(1.0, 2.0, 3.0), meta["$JSON_ARRAY_KEY[2"].value?.list?.map { it.number.toDouble() })
|
||||
assertEquals(true, meta["${Meta.JSON_ARRAY_KEY}[0].${Meta.JSON_ARRAY_KEY}[1].d"].boolean)
|
||||
assertEquals("value", meta["${Meta.JSON_ARRAY_KEY}[1]"].string)
|
||||
assertEquals(listOf(1.0, 2.0, 3.0), meta["${Meta.JSON_ARRAY_KEY}[2]"]?.value?.list?.map { it.double })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testJsonStringToMeta(){
|
||||
fun testJsonStringToMeta() {
|
||||
val jsonString = """
|
||||
{
|
||||
"comments": [
|
||||
@ -98,8 +82,8 @@ class MetaFormatTest {
|
||||
}
|
||||
""".trimIndent()
|
||||
val json = Json.parseToJsonElement(jsonString)
|
||||
val meta = json.toMetaItem().node!!
|
||||
assertEquals(ListValue.EMPTY, meta["comments"].value)
|
||||
val meta = json.toMeta()
|
||||
assertEquals(ListValue.EMPTY, meta["comments"]?.value)
|
||||
}
|
||||
|
||||
}
|
@ -5,9 +5,8 @@ import kotlinx.serialization.cbor.Cbor
|
||||
import kotlinx.serialization.json.Json
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.MetaSerializer
|
||||
import space.kscience.dataforge.meta.TypedMetaItem
|
||||
import space.kscience.dataforge.meta.seal
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.toName
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
@ -28,7 +27,7 @@ class MetaSerializerTest {
|
||||
fun testMetaSerialization() {
|
||||
val string = JSON_PRETTY.encodeToString(MetaSerializer, meta)
|
||||
println(string)
|
||||
val restored = JSON_PLAIN.decodeFromString(MetaSerializer, string)
|
||||
val restored = JSON_PLAIN.decodeFromString(MetaSerializer, string).seal()
|
||||
assertEquals(meta, restored)
|
||||
}
|
||||
|
||||
@ -43,7 +42,7 @@ class MetaSerializerTest {
|
||||
|
||||
@Test
|
||||
fun testNameSerialization() {
|
||||
val name = "a.b.c".toName()
|
||||
val name = Name.parse("a.b.c")
|
||||
val string = JSON_PRETTY.encodeToString(Name.serializer(), name)
|
||||
val restored = JSON_PLAIN.decodeFromString(Name.serializer(), string)
|
||||
assertEquals(name, restored)
|
||||
@ -52,7 +51,7 @@ class MetaSerializerTest {
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
@Test
|
||||
fun testMetaItemDescriptor() {
|
||||
val descriptor = TypedMetaItem.serializer(MetaSerializer).descriptor.getElementDescriptor(0)
|
||||
val descriptor = MetaSerializer.descriptor.getElementDescriptor(0)
|
||||
println(descriptor)
|
||||
}
|
||||
}
|
@ -1,5 +1,6 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.writeString
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.meta.get
|
||||
import space.kscience.dataforge.meta.int
|
||||
@ -18,32 +19,31 @@ class MultipartTest {
|
||||
"value" put it
|
||||
}
|
||||
data {
|
||||
writeUtf8String("Hello World $it")
|
||||
writeString("Hello World $it")
|
||||
repeat(300) {
|
||||
writeRawString("$it ")
|
||||
writeString("$it ")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val partsEnvelope = Envelope {
|
||||
envelopes(envelopes, TaglessEnvelopeFormat)
|
||||
envelopes(envelopes)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun testParts() {
|
||||
TaglessEnvelopeFormat.run {
|
||||
val singleEnvelopeData = toBinary(envelopes[0])
|
||||
val singleEnvelopeSize = singleEnvelopeData.size
|
||||
val bytes = toBinary(partsEnvelope)
|
||||
assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
|
||||
val reconstructed = bytes.readWith(this)
|
||||
println(reconstructed.meta)
|
||||
val parts = reconstructed.parts()
|
||||
val envelope = parts[2].envelope(io)
|
||||
assertEquals(2, envelope.meta["value"].int)
|
||||
println(reconstructed.data!!.size)
|
||||
}
|
||||
val format = TaggedEnvelopeFormat
|
||||
val singleEnvelopeData = Binary(envelopes[0], format)
|
||||
val singleEnvelopeSize = singleEnvelopeData.size
|
||||
val bytes = Binary(partsEnvelope, format)
|
||||
assertTrue(envelopes.size * singleEnvelopeSize < bytes.size)
|
||||
val reconstructed = bytes.readWith(format)
|
||||
println(reconstructed.meta)
|
||||
val parts = reconstructed.parts()
|
||||
val envelope = parts[2].envelope()
|
||||
assertEquals(2, envelope.meta["value"].int)
|
||||
println(reconstructed.data!!.size)
|
||||
}
|
||||
|
||||
}
|
@ -1,12 +1,11 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.ByteReadPacket
|
||||
import io.ktor.utils.io.core.use
|
||||
import kotlinx.io.buffered
|
||||
|
||||
|
||||
fun <T : Any> IOFormat<T>.writeToByteArray(obj: T): ByteArray = buildByteArray {
|
||||
writeObject(this, obj)
|
||||
fun <T : Any> IOFormat<T>.writeToByteArray(obj: T): ByteArray = ByteArray {
|
||||
writeTo(this, obj)
|
||||
}
|
||||
fun <T : Any> IOFormat<T>.readFromByteArray(array: ByteArray): T = ByteReadPacket(array).use {
|
||||
readObject(it)
|
||||
fun <T : Any> IOFormat<T>.readFromByteArray(array: ByteArray): T = ByteArraySource(array).buffered().use {
|
||||
readFrom(it)
|
||||
}
|
@ -1,15 +1,18 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.*
|
||||
import io.ktor.utils.io.streams.asOutput
|
||||
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.io.Sink
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asSink
|
||||
import kotlinx.io.buffered
|
||||
import space.kscience.dataforge.meta.Meta
|
||||
import space.kscience.dataforge.meta.descriptors.NodeDescriptor
|
||||
import space.kscience.dataforge.meta.descriptors.MetaDescriptor
|
||||
import space.kscience.dataforge.meta.isEmpty
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardOpenOption
|
||||
import kotlin.io.path.ExperimentalPathApi
|
||||
import kotlin.io.path.inputStream
|
||||
import kotlin.math.min
|
||||
import kotlin.reflect.full.isSupertypeOf
|
||||
@ -23,86 +26,114 @@ internal class PathBinary(
|
||||
override val size: Int = Files.size(path).toInt() - fileOffset,
|
||||
) : Binary {
|
||||
|
||||
@OptIn(ExperimentalPathApi::class)
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Input.() -> R): R {
|
||||
override fun <R> read(offset: Int, atMost: Int, block: Source.() -> R): R = runBlocking {
|
||||
readSuspend(offset, atMost, block)
|
||||
}
|
||||
|
||||
override suspend fun <R> readSuspend(offset: Int, atMost: Int, block: suspend Source.() -> R): R {
|
||||
val actualOffset = offset + fileOffset
|
||||
val actualSize = min(atMost, size - offset)
|
||||
val array = path.inputStream().use {
|
||||
it.skip(actualOffset.toLong())
|
||||
it.readNBytes(actualSize)
|
||||
}
|
||||
return ByteReadPacket(array).block()
|
||||
return ByteArraySource(array).buffered().use { it.block() }
|
||||
}
|
||||
|
||||
override fun view(offset: Int, binarySize: Int) = PathBinary(path, fileOffset + offset, binarySize)
|
||||
}
|
||||
|
||||
public fun Path.asBinary(): Binary = PathBinary(this)
|
||||
|
||||
public fun <R> Path.read(block: Input.() -> R): R = asBinary().read(block = block)
|
||||
public fun <R> Path.read(block: Source.() -> R): R = asBinary().read(block = block)
|
||||
|
||||
/**
|
||||
* Write a live output to a newly created file. If file does not exist, throws error
|
||||
*/
|
||||
public fun Path.write(block: Output.() -> Unit): Unit {
|
||||
public fun Path.write(block: Sink.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(this, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW)
|
||||
stream.asOutput().use(block)
|
||||
stream.asSink().buffered().use(block)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new file or append to exiting one with given output [block]
|
||||
*/
|
||||
public fun Path.append(block: Output.() -> Unit): Unit {
|
||||
public fun Path.append(block: Sink.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(
|
||||
this,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE
|
||||
)
|
||||
stream.asOutput().use(block)
|
||||
stream.asSink().buffered().use(block)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new file or replace existing one using given output [block]
|
||||
*/
|
||||
public fun Path.rewrite(block: Output.() -> Unit): Unit {
|
||||
public fun Path.rewrite(block: Sink.() -> Unit): Unit {
|
||||
val stream = Files.newOutputStream(
|
||||
this,
|
||||
StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE
|
||||
)
|
||||
stream.asOutput().use(block)
|
||||
stream.asSink().buffered().use(block)
|
||||
}
|
||||
|
||||
public fun Path.readEnvelope(format: EnvelopeFormat): Envelope {
|
||||
val partialEnvelope: PartialEnvelope = asBinary().read {
|
||||
format.run {
|
||||
readPartial(this@read)
|
||||
}
|
||||
}
|
||||
val offset: Int = partialEnvelope.dataOffset.toInt()
|
||||
val size: Int = partialEnvelope.dataSize?.toInt() ?: (Files.size(this).toInt() - offset)
|
||||
val binary = PathBinary(this, offset, size)
|
||||
return SimpleEnvelope(partialEnvelope.meta, binary)
|
||||
}
|
||||
@DFExperimental
|
||||
public fun EnvelopeFormat.readFile(path: Path): Envelope = readFrom(path.asBinary())
|
||||
|
||||
/**
|
||||
* Resolve IOFormat based on type
|
||||
*/
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
@DFExperimental
|
||||
public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? {
|
||||
return ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>?
|
||||
public inline fun <reified T : Any> IOPlugin.resolveIOFormat(): IOFormat<T>? =
|
||||
ioFormatFactories.find { it.type.isSupertypeOf(typeOf<T>()) } as IOFormat<T>?
|
||||
|
||||
|
||||
public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
|
||||
public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
|
||||
|
||||
/**
|
||||
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
|
||||
* If [path] is a directory search for file starting with `meta` in it.
|
||||
*
|
||||
* Returns null if meta could not be resolved
|
||||
*/
|
||||
public fun IOPlugin.readMetaFileOrNull(
|
||||
path: Path,
|
||||
formatOverride: MetaFormat? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
): Meta? {
|
||||
if (!Files.exists(path)) return null
|
||||
|
||||
val actualPath: Path = if (Files.isDirectory(path)) {
|
||||
Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) }
|
||||
?: return null
|
||||
} else {
|
||||
path
|
||||
}
|
||||
val extension = actualPath.fileName.toString().substringAfterLast('.')
|
||||
|
||||
val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: return null
|
||||
return actualPath.read {
|
||||
metaFormat.readMeta(this, descriptor)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read file containing meta using given [formatOverride] or file extension to infer meta type.
|
||||
* If [path] is a directory search for file starting with `meta` in it
|
||||
* If [path] is a directory search for file starting with `meta` in it.
|
||||
*
|
||||
* Fails if nothing works.
|
||||
*/
|
||||
public fun IOPlugin.readMetaFile(
|
||||
path: Path,
|
||||
formatOverride: MetaFormat? = null,
|
||||
descriptor: NodeDescriptor? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
): Meta {
|
||||
if (!Files.exists(path)) error("Meta file $path does not exist")
|
||||
|
||||
val actualPath: Path = if (Files.isDirectory(path)) {
|
||||
Files.list(path).asSequence().find { it.fileName.startsWith("meta") }
|
||||
Files.list(path).asSequence().find { it.fileName.startsWith(IOPlugin.META_FILE_NAME) }
|
||||
?: error("The directory $path does not contain meta file")
|
||||
} else {
|
||||
path
|
||||
@ -110,13 +141,12 @@ public fun IOPlugin.readMetaFile(
|
||||
val extension = actualPath.fileName.toString().substringAfterLast('.')
|
||||
|
||||
val metaFormat = formatOverride ?: resolveMetaFormat(extension) ?: error("Can't resolve meta format $extension")
|
||||
return metaFormat.run {
|
||||
actualPath.read {
|
||||
readMeta(this, descriptor)
|
||||
}
|
||||
return actualPath.read {
|
||||
metaFormat.readMeta(this, descriptor)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Write meta to file using [metaFormat]. If [path] is a directory, write a file with name equals name of [metaFormat].
|
||||
* Like "meta.json"
|
||||
@ -125,7 +155,7 @@ public fun IOPlugin.writeMetaFile(
|
||||
path: Path,
|
||||
meta: Meta,
|
||||
metaFormat: MetaFormatFactory = JsonMetaFormat,
|
||||
descriptor: NodeDescriptor? = null,
|
||||
descriptor: MetaDescriptor? = null,
|
||||
) {
|
||||
val actualPath = if (Files.isDirectory(path)) {
|
||||
path.resolve("@" + metaFormat.name.toString())
|
||||
@ -145,19 +175,9 @@ public fun IOPlugin.writeMetaFile(
|
||||
*/
|
||||
public fun IOPlugin.peekFileEnvelopeFormat(path: Path): EnvelopeFormat? {
|
||||
val binary = path.asBinary()
|
||||
val formats = envelopeFormatFactories.mapNotNull { factory ->
|
||||
factory.peekFormat(this@peekFileEnvelopeFormat, binary)
|
||||
}
|
||||
|
||||
return when (formats.size) {
|
||||
0 -> null
|
||||
1 -> formats.first()
|
||||
else -> error("Envelope format binary recognition clash: $formats")
|
||||
}
|
||||
return peekBinaryEnvelopeFormat(binary)
|
||||
}
|
||||
|
||||
public val IOPlugin.Companion.META_FILE_NAME: String get() = "@meta"
|
||||
public val IOPlugin.Companion.DATA_FILE_NAME: String get() = "@data"
|
||||
|
||||
/**
|
||||
* Read and envelope from file if the file exists, return null if file does not exist.
|
||||
@ -203,22 +223,11 @@ public fun IOPlugin.readEnvelopeFile(
|
||||
return SimpleEnvelope(meta, data)
|
||||
}
|
||||
|
||||
return formatPicker(path)?.let { format ->
|
||||
path.readEnvelope(format)
|
||||
} ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
|
||||
return formatPicker(path)?.readFile(path) ?: if (readNonEnvelopes) { // if no format accepts file, read it as binary
|
||||
SimpleEnvelope(Meta.EMPTY, path.asBinary())
|
||||
} else error("Can't infer format for file $path")
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a binary into file. Throws an error if file already exists
|
||||
*/
|
||||
public fun <T : Any> IOFormat<T>.writeToFile(path: Path, obj: T) {
|
||||
path.write {
|
||||
writeObject(this, obj)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write envelope file to given [path] using [envelopeFormat] and optional [metaFormat]
|
||||
*/
|
||||
@ -227,10 +236,9 @@ public fun IOPlugin.writeEnvelopeFile(
|
||||
path: Path,
|
||||
envelope: Envelope,
|
||||
envelopeFormat: EnvelopeFormat = TaggedEnvelopeFormat,
|
||||
metaFormat: MetaFormatFactory? = null,
|
||||
) {
|
||||
path.rewrite {
|
||||
envelopeFormat.writeEnvelope(this, envelope, metaFormat ?: envelopeFormat.defaultMetaFormat)
|
||||
envelopeFormat.writeTo(this, envelope)
|
||||
}
|
||||
}
|
||||
|
||||
@ -255,7 +263,7 @@ public fun IOPlugin.writeEnvelopeDirectory(
|
||||
val dataFile = path.resolve(IOPlugin.DATA_FILE_NAME)
|
||||
dataFile.write {
|
||||
envelope.data?.read {
|
||||
val copied = copyTo(this@write)
|
||||
val copied = transferTo(this@write)
|
||||
if (copied != envelope.data?.size?.toLong()) {
|
||||
error("The number of copied bytes does not equal data size")
|
||||
}
|
||||
|
@ -0,0 +1,11 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import kotlinx.io.Source
|
||||
import kotlinx.io.asSource
|
||||
import kotlinx.io.buffered
|
||||
|
||||
|
||||
public fun IOPlugin.resource(name: String): Binary? = { }.javaClass.getResource(name)?.readBytes()?.asBinary()
|
||||
|
||||
public inline fun <R> IOPlugin.readResource(name: String, block: Source.() -> R): R =
|
||||
{ }.javaClass.getResource(name)?.openStream()?.asSource()?.buffered()?.block() ?: error("Can't read resource $name")
|
@ -0,0 +1,27 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import space.kscience.dataforge.context.ContextBuilder
|
||||
import space.kscience.dataforge.meta.set
|
||||
import space.kscience.dataforge.meta.string
|
||||
import java.nio.file.Path
|
||||
import kotlin.io.path.Path
|
||||
|
||||
|
||||
public val IOPlugin.workDirectory: Path
|
||||
get() {
|
||||
val workDirectoryPath = meta[IOPlugin.WORK_DIRECTORY_KEY].string
|
||||
?: context.properties[IOPlugin.WORK_DIRECTORY_KEY].string
|
||||
?: ".dataforge"
|
||||
|
||||
return Path(workDirectoryPath)
|
||||
}
|
||||
|
||||
public fun ContextBuilder.workDirectory(path: String) {
|
||||
properties {
|
||||
set(IOPlugin.WORK_DIRECTORY_KEY, path)
|
||||
}
|
||||
}
|
||||
|
||||
public fun ContextBuilder.workDirectory(path: Path) {
|
||||
workDirectory(path.toAbsolutePath().toString())
|
||||
}
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
|
@ -1,6 +1,5 @@
|
||||
package space.kscience.dataforge.io
|
||||
|
||||
import io.ktor.utils.io.core.writeDouble
|
||||
import space.kscience.dataforge.context.Global
|
||||
import space.kscience.dataforge.misc.DFExperimental
|
||||
import java.nio.file.Files
|
||||
|
23
dataforge-meta/README.md
Normal file
23
dataforge-meta/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Module dataforge-meta
|
||||
|
||||
Meta definition and basic operations on meta
|
||||
|
||||
## Usage
|
||||
|
||||
## Artifact:
|
||||
|
||||
The Maven coordinates of this project are `space.kscience:dataforge-meta:0.7.0`.
|
||||
|
||||
**Gradle Kotlin DSL:**
|
||||
```kotlin
|
||||
repositories {
|
||||
maven("https://repo.kotlin.link")
|
||||
//uncomment to access development builds
|
||||
//maven("https://maven.pkg.jetbrains.space/spc/p/sci/dev")
|
||||
mavenCentral()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("space.kscience:dataforge-meta:0.7.0")
|
||||
}
|
||||
```
|
File diff suppressed because it is too large
Load Diff
@ -1,9 +1,11 @@
|
||||
plugins {
|
||||
id("ru.mipt.npm.gradle.mpp")
|
||||
id("ru.mipt.npm.gradle.native")
|
||||
id("space.kscience.gradle.mpp")
|
||||
}
|
||||
|
||||
kscience {
|
||||
jvm()
|
||||
js()
|
||||
native()
|
||||
useSerialization{
|
||||
json()
|
||||
}
|
||||
@ -12,5 +14,5 @@ kscience {
|
||||
description = "Meta definition and basic operations on meta"
|
||||
|
||||
readme{
|
||||
maturity = ru.mipt.npm.gradle.Maturity.DEVELOPMENT
|
||||
maturity = space.kscience.gradle.Maturity.DEVELOPMENT
|
||||
}
|
@ -1,110 +0,0 @@
|
||||
package space.kscience.dataforge.meta
|
||||
|
||||
import kotlinx.serialization.KSerializer
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.descriptors.SerialDescriptor
|
||||
import kotlinx.serialization.encoding.Decoder
|
||||
import kotlinx.serialization.encoding.Encoder
|
||||
import space.kscience.dataforge.names.Name
|
||||
import space.kscience.dataforge.names.NameToken
|
||||
import space.kscience.dataforge.names.asName
|
||||
import space.kscience.dataforge.names.plus
|
||||
import kotlin.collections.set
|
||||
import kotlin.jvm.Synchronized
|
||||
|
||||
//TODO add validator to configuration
|
||||
|
||||
/**
|
||||
* Mutable meta representing object state
|
||||
*/
|
||||
@Serializable(Config.Companion::class)
|
||||
public class Config : AbstractMutableMeta<Config>(), ItemPropertyProvider {
|
||||
|
||||
private val listeners = HashSet<ItemListener>()
|
||||
|
||||
@Synchronized
|
||||
private fun itemChanged(name: Name, oldItem: MetaItem?, newItem: MetaItem?) {
|
||||
listeners.forEach { it.action(name, oldItem, newItem) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Add change listener to this meta. Owner is declared to be able to remove listeners later. Listener without owner could not be removed
|
||||
*/
|
||||
@Synchronized
|
||||
override fun onChange(owner: Any?, action: (Name, MetaItem?, MetaItem?) -> Unit) {
|
||||
listeners.add(ItemListener(owner, action))
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all listeners belonging to given owner
|
||||
*/
|
||||
@Synchronized
|
||||
override fun removeListener(owner: Any?) {
|
||||
listeners.removeAll { it.owner === owner }
|
||||
}
|
||||
|
||||
override fun replaceItem(key: NameToken, oldItem: TypedMetaItem<Config>?, newItem: TypedMetaItem<Config>?) {
|
||||
if (newItem == null) {
|
||||
children.remove(key)
|
||||
if (oldItem != null && oldItem is MetaItemNode<Config>) {
|
||||
oldItem.node.removeListener(this)
|
||||
}
|
||||
} else {
|
||||
children[key] = newItem
|
||||
if (newItem is MetaItemNode) {
|
||||
newItem.node.onChange(this) { name, oldChild, newChild ->
|
||||
itemChanged(key + name, oldChild, newChild)
|
||||
}
|
||||
}
|
||||
}
|
||||
itemChanged(key.asName(), oldItem, newItem)
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach configuration node instead of creating one
|
||||
*/
|
||||
override fun wrapNode(meta: Meta): Config = meta.asConfig()
|
||||
|
||||
override fun empty(): Config = Config()
|
||||
|
||||
public companion object ConfigSerializer : KSerializer<Config> {
|
||||
|
||||
public fun empty(): Config = Config()
|
||||
override val descriptor: SerialDescriptor get() = MetaSerializer.descriptor
|
||||
|
||||
override fun deserialize(decoder: Decoder): Config {
|
||||
return MetaSerializer.deserialize(decoder).asConfig()
|
||||
}
|
||||
|
||||
override fun serialize(encoder: Encoder, value: Config) {
|
||||
MetaSerializer.serialize(encoder, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public operator fun Config.get(token: NameToken): TypedMetaItem<Config>? = items[token]
|
||||
|
||||
/**
|
||||
* Create a mutable copy of this [Meta]. The copy is created event if initial [Meta] is a [Config].
|
||||
* Listeners are not preserved
|
||||
*/
|
||||
public fun Meta.toConfig(): Config = Config().also { builder ->
|
||||
this.items.mapValues { entry ->
|
||||
val item = entry.value
|
||||
builder[entry.key.asName()] = when (item) {
|
||||
is MetaItemValue -> item.value
|
||||
is MetaItemNode -> MetaItemNode(item.node.asConfig())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a copy of this config, optionally applying the given [block].
|
||||
* The listeners of the original Config are not retained.
|
||||
*/
|
||||
public inline fun Config.copy(block: Config.() -> Unit = {}): Config = toConfig().apply(block)
|
||||
|
||||
/**
|
||||
* Return this [Meta] as [Config] if it is [Config] and create a new copy otherwise
|
||||
*/
|
||||
public fun Meta.asConfig(): Config = this as? Config ?: toConfig()
|
@ -1,25 +1,19 @@
|
||||
package space.kscience.dataforge.meta
|
||||
|
||||
import space.kscience.dataforge.misc.DFBuilder
|
||||
import space.kscience.dataforge.names.Name
|
||||
import kotlin.properties.ReadWriteProperty
|
||||
|
||||
/**
|
||||
* A container that holds a [Config].
|
||||
* A container that holds a [ObservableMeta].
|
||||
*/
|
||||
public interface Configurable {
|
||||
/**
|
||||
* Backing config
|
||||
*/
|
||||
public val config: Config
|
||||
public val meta: MutableMeta
|
||||
}
|
||||
|
||||
|
||||
public fun <T : Configurable> T.configure(meta: Meta): T = this.apply { config.update(meta) }
|
||||
public fun <T : Configurable> T.configure(meta: Meta): T = this.apply { this.meta.update(meta) }
|
||||
|
||||
@DFBuilder
|
||||
public inline fun <T : Configurable> T.configure(action: Config.() -> Unit): T = apply { config.apply(action) }
|
||||
|
||||
/* Node delegates */
|
||||
|
||||
public fun Configurable.config(key: Name? = null): ReadWriteProperty<Any?, Config?> = config.node(key)
|
||||
public inline fun <T : Configurable> T.configure(action: MutableMeta.() -> Unit): T = apply { meta.apply(action) }
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user