From d67a74ccd03c965fc1b6fe315d0a8493c5790845 Mon Sep 17 00:00:00 2001 From: Alexander Nozik Date: Sat, 16 Apr 2016 22:41:21 +0300 Subject: [PATCH] implemented file object storage --- .../inr/numass/scripts/TestClient.groovy | 4 +- .../inr/numass/scripts/TestNotes.groovy | 20 ++++ .../java/inr/numass/client/NumassClient.java | 42 ++++++-- .../inr/numass/scripts/TestServer.groovy | 2 +- .../java/inr/numass/server/NumassNote.java | 77 +++++++++++++++ .../java/inr/numass/server/NumassRun.java | 97 ++++++++++++++++++- .../java/inr/numass/server/NumassServer.java | 9 +- .../numass/server/NumassStorageHandler.java | 89 +++++++++++++++++ .../server/UnknownNumassActionException.java | 64 ++++++------ .../main/resources/templates/NoteLoader.ftl | 14 +++ .../inr/numass/storage/NumassDataLoader.java | 4 +- 11 files changed, 371 insertions(+), 51 deletions(-) create mode 100644 numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestNotes.groovy create mode 100644 numass-storage/numass-server/src/main/java/inr/numass/server/NumassNote.java create mode 100644 numass-storage/numass-server/src/main/java/inr/numass/server/NumassStorageHandler.java create mode 100644 numass-storage/numass-server/src/main/resources/templates/NoteLoader.ftl diff --git a/numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestClient.groovy b/numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestClient.groovy index 0d3013dc..599e5842 100644 --- a/numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestClient.groovy +++ b/numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestClient.groovy @@ -26,8 +26,6 @@ import inr.numass.client.NumassClient new StorageManager().startGlobal(); - - MetaStreamWriter parser = new JSONMetaWriter(); println "Starting Numass test client..." @@ -38,7 +36,7 @@ BufferedReader br = new BufferedReader(new InputStreamReader(System.in)) while(line == null || !line.startsWith("exit")){ // print ">" line = br.readLine(); - if(!line.startsWith("exit")){ + if(line!= null && !line.startsWith("exit")){ NumassClient.runComand("127.0.0.1", 8335, line.split(" ")); } } diff --git a/numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestNotes.groovy b/numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestNotes.groovy new file mode 100644 index 00000000..1e092640 --- /dev/null +++ b/numass-storage/numass-client/src/main/groovy/inr/numass/scripts/TestNotes.groovy @@ -0,0 +1,20 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ + +package inr.numass.scripts + +import hep.dataforge.io.MetaStreamReader +import hep.dataforge.io.MetaStreamWriter +import hep.dataforge.meta.Meta +import hep.dataforge.storage.commons.JSONMetaWriter +import hep.dataforge.storage.commons.StorageManager +import inr.numass.client.NumassClient + +new StorageManager().startGlobal(); + +MetaStreamWriter parser = new JSONMetaWriter(); + +NumassClient.runComand("127.0.0.1", 8336, "addNote", "This is my note with html"); diff --git a/numass-storage/numass-client/src/main/java/inr/numass/client/NumassClient.java b/numass-storage/numass-client/src/main/java/inr/numass/client/NumassClient.java index 11e69573..5d91c7dd 100644 --- a/numass-storage/numass-client/src/main/java/inr/numass/client/NumassClient.java +++ b/numass-storage/numass-client/src/main/java/inr/numass/client/NumassClient.java @@ -36,6 +36,7 @@ import java.net.Socket; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.StandardOpenOption; +import java.time.Instant; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -203,6 +204,23 @@ public class NumassClient extends MessageFactory implements Closeable { return sendAndRecieve(env.build()).meta(); } + public Meta addNote(String text, Instant time) { + EnvelopeBuilder env = requestActionBase("numass.notes", "push"); + env.putMetaValue("note.text", text); + if (time != null) { + env.putMetaValue("note.time", time); + } + return sendAndRecieve(env.build()).meta(); + } + + public Meta getNotes(int limit) { + EnvelopeBuilder env = requestActionBase("numass.notes", "pull"); + if (limit > 0) { + env.putMetaValue("limit", limit); + } + return sendAndRecieve(env.build()).meta(); + } + public static void main(String[] args) { new StorageManager().startGlobal(); @@ -224,8 +242,8 @@ public class NumassClient extends MessageFactory implements Closeable { } - public static void runComand(String ip, int port, String[] args) { - checkArgLength(args, 1); + public static void runComand(String ip, int port, String... args) { + checkArgLength(1, args); try (NumassClient client = new NumassClient(ip, port)) { switch (args[0]) { case "getRun": @@ -237,7 +255,7 @@ public class NumassClient extends MessageFactory implements Closeable { } return; case "setRun": - checkArgLength(args, 2); + checkArgLength(2, args); Meta setRun = client.startRun(args[1]); if (setRun.getBoolean("success", true)) { System.out.println(setRun.getString("run.path")); @@ -246,7 +264,7 @@ public class NumassClient extends MessageFactory implements Closeable { } return; case "getState": - checkArgLength(args, 2); + checkArgLength(2, args); String stateName = args[1]; Map states = client.getStates(stateName); if (states != null) { @@ -256,7 +274,7 @@ public class NumassClient extends MessageFactory implements Closeable { } return; case "setState": - checkArgLength(args, 3); + checkArgLength(3, args); String setStateName = args[1]; String setStateValue = args[2]; Meta setStateMeta = client.setState(setStateName, setStateValue); @@ -267,7 +285,7 @@ public class NumassClient extends MessageFactory implements Closeable { } return; case "pushPoint": - checkArgLength(args, 2); + checkArgLength(2, args); String path; String fileName; if (args.length == 2) { @@ -285,6 +303,16 @@ public class NumassClient extends MessageFactory implements Closeable { } else { System.out.println("Error: operaton failed"); } + return; + case "addNote": + checkArgLength(2, args); + String note = args[1]; + Meta addNote = client.addNote(note, null); + if (addNote.getBoolean("success", true)) { + System.out.println("OK"); + } else { + System.out.println("Error: operaton failed"); + } } } catch (IOException ex) { @@ -294,7 +322,7 @@ public class NumassClient extends MessageFactory implements Closeable { } } - private static void checkArgLength(String[] args, int length) { + private static void checkArgLength(int length, String... args) { if (args.length < length) { LoggerFactory.getLogger("NumassClient").error("Command line to short"); System.exit(1); diff --git a/numass-storage/numass-server/src/main/groovy/inr/numass/scripts/TestServer.groovy b/numass-storage/numass-server/src/main/groovy/inr/numass/scripts/TestServer.groovy index cb7a29c4..19865fc9 100644 --- a/numass-storage/numass-server/src/main/groovy/inr/numass/scripts/TestServer.groovy +++ b/numass-storage/numass-server/src/main/groovy/inr/numass/scripts/TestServer.groovy @@ -18,7 +18,7 @@ package inr.numass.scripts import hep.dataforge.storage.filestorage.FileStorage import inr.numass.server.NumassServer -String path = "D:\\temp\\test\\numass\\" +String path = "D:\\temp\\test\\numass-server\\" FileStorage storage = FileStorage.in(new File(path), null); diff --git a/numass-storage/numass-server/src/main/java/inr/numass/server/NumassNote.java b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassNote.java new file mode 100644 index 00000000..17204d0d --- /dev/null +++ b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassNote.java @@ -0,0 +1,77 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package inr.numass.server; + +import hep.dataforge.meta.Meta; +import hep.dataforge.meta.MetaBuilder; +import java.io.Serializable; +import java.time.Instant; + +/** + * A progress note for numass run + * + * @author Alexander Nozik + */ +public class NumassNote implements Serializable { + + public static NumassNote buildFrom(Meta meta) { + String text = meta.getString("text", ""); + if (meta.hasValue("time")) { + Instant time = meta.getValue("time").timeValue(); + return new NumassNote(text, time); + } else { + return new NumassNote(text); + } + } + + private final String content; + private final String ref; + private final Instant time; + + public NumassNote(String content, Instant time) { + this.content = content; + this.time = time; + this.ref = "#" + time.hashCode(); + } + + public NumassNote(String content) { + this.content = content; + this.time = Instant.now(); + this.ref = "#" + time.hashCode(); + } + + /** + * Text content + * + * @return + */ + public String content() { + return content; + } + + /** + * + * @return + */ + public Instant time() { + return time; + } + + /** + * Unique note name for references + * + * @return + */ + public String ref() { + return ref; + } + + public Meta toMeta() { + return new MetaBuilder("note").putValue("time", time) + .putValue("ref", ref) + .putValue("text", content); + } +} diff --git a/numass-storage/numass-server/src/main/java/inr/numass/server/NumassRun.java b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassRun.java index 2fdf47fa..218b5c96 100644 --- a/numass-storage/numass-server/src/main/java/inr/numass/server/NumassRun.java +++ b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassRun.java @@ -18,6 +18,7 @@ package inr.numass.server; import hep.dataforge.data.binary.Binary; import hep.dataforge.exceptions.StorageException; import hep.dataforge.io.envelopes.Envelope; +import hep.dataforge.io.envelopes.EnvelopeBuilder; import hep.dataforge.io.envelopes.Responder; import hep.dataforge.meta.Annotated; import hep.dataforge.meta.Meta; @@ -25,8 +26,15 @@ import hep.dataforge.storage.api.StateLoader; import hep.dataforge.storage.commons.LoaderFactory; import hep.dataforge.storage.commons.MessageFactory; import hep.dataforge.values.Value; -import inr.numass.storage.NumassStorage; import java.io.IOException; +import java.time.Instant; +import java.util.stream.Stream; +import hep.dataforge.storage.api.ObjectLoader; +import inr.numass.storage.NumassStorage; +import java.util.Comparator; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This object governs remote access to numass storage and performs reading and @@ -37,6 +45,7 @@ import java.io.IOException; public class NumassRun implements Annotated, Responder { public static final String RUN_STATE = "@run"; + public static final String RUN_NOTES = "@notes"; private final String runPath; /** @@ -48,13 +57,23 @@ public class NumassRun implements Annotated, Responder { * Default state loader for this run */ private final StateLoader states; + + private final ObjectLoader noteLoader; private final MessageFactory factory; + private final Logger logger; + +// /** +// * A set with inverted order of elements (last note first) +// */ +// private final Set notes = new TreeSet<>((NumassNote o1, NumassNote o2) -> -o1.time().compareTo(o2.time())); public NumassRun(String path, NumassStorage workStorage, MessageFactory factory) throws StorageException { this.storage = workStorage; this.states = LoaderFactory.buildStateLoder(storage, RUN_STATE, null); + this.noteLoader = LoaderFactory.buildObjectLoder(storage, RUN_NOTES, null); this.factory = factory; this.runPath = path; + logger = LoggerFactory.getLogger("CURRENT_RUN"); } public Value getState(String name) { @@ -88,12 +107,85 @@ public class NumassRun implements Annotated, Responder { default: throw new UnknownNumassActionException(action, UnknownNumassActionException.Cause.NOT_SUPPORTED); } + case "numass.notes": + switch (action) { + case "push": + return pushNote(message); + case "pull": + return pullNotes(message); + default: + throw new UnknownNumassActionException(action, UnknownNumassActionException.Cause.NOT_SUPPORTED); + } + default: throw new RuntimeException("Wrong message type"); } } - public Envelope pushNumassPoint(Envelope message) { + public synchronized void addNote(String text, Instant time) throws StorageException { + NumassNote note = new NumassNote(text, time); + addNote(note); + } + + @SuppressWarnings("unchecked") + public synchronized void addNote(NumassNote note) throws StorageException { + noteLoader.push(note.ref(), note); + } + + /** + * Stream of notes in the last to first order + * + * @return + */ + @SuppressWarnings("unchecked") + public Stream getNotes() { + return noteLoader.fragmentNames().stream().map(new Function() { + @Override + public NumassNote apply(String name) { + try { + return (NumassNote) noteLoader.pull(name); + } catch (StorageException ex) { + return (NumassNote) null; + } + } + }).sorted(new Comparator() { + @Override + public int compare(NumassNote o1, NumassNote o2) { + return -o1.time().compareTo(o2.time()); + } + }); + } + + private synchronized Envelope pushNote(Envelope message) { + try { + if (message.meta().hasNode("note")) { + for (Meta node : message.meta().getNodes("note")) { + addNote(NumassNote.buildFrom(node)); + } + } else { + addNote(NumassNote.buildFrom(message.meta())); + } + return factory.okResponseBase(message, false, false).build(); + } catch (Exception ex) { + logger.error("Failed to push note", ex); + return factory.errorResponseBase(message, ex).build(); + } + } + + private Envelope pullNotes(Envelope message) { + EnvelopeBuilder envelope = factory.okResponseBase(message, true, false); + int limit = message.meta().getInt("limit", -1); + //TODO add time window and search conditions here + Stream stream = getNotes(); + if (limit > 0) { + stream = stream.limit(limit); + } + stream.forEach((NumassNote note) -> envelope.putMetaNode(note.toMeta())); + + return envelope.build(); + } + + private Envelope pushNumassPoint(Envelope message) { try { String filePath = message.meta().getString("path", ""); String fileName = message.meta().getString("name") @@ -102,6 +194,7 @@ public class NumassRun implements Annotated, Responder { //TODO add checksum here return factory.okResponseBase("numass.data.push.response", false, false).build(); } catch (StorageException | IOException ex) { + logger.error("Failed to push point", ex); return factory.errorResponseBase("numass.data.push.response", ex).build(); } } diff --git a/numass-storage/numass-server/src/main/java/inr/numass/server/NumassServer.java b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassServer.java index bb3fa2d3..4e4c63d8 100644 --- a/numass-storage/numass-server/src/main/java/inr/numass/server/NumassServer.java +++ b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassServer.java @@ -24,7 +24,7 @@ import hep.dataforge.storage.commons.AbstractNetworkListener; import hep.dataforge.storage.commons.LoaderFactory; import hep.dataforge.storage.commons.StorageManager; import hep.dataforge.storage.filestorage.FileStorage; -import hep.dataforge.storage.servlet.SorageRatpackHandler; +import hep.dataforge.storage.servlet.StorageRatpackHandler; import inr.numass.storage.NumassStorage; import java.io.File; import java.io.IOException; @@ -84,7 +84,7 @@ public class NumassServer extends AbstractNetworkListener { .serverConfig((ServerConfigBuilder config) -> config.port(port)) .handlers((Chain chain) -> chain .get(new NumassRootHandler(this)) - .get("storage", new SorageRatpackHandler(root)) + .get("storage", new NumassStorageHandler(root)) ) ); } @@ -114,6 +114,7 @@ public class NumassServer extends AbstractNetworkListener { case "numass.state": return getRootState().respond(message); case "numass.data": + case "numass.notes": case "numass.run.state": return getRun().respond(message); case "numass.control": @@ -282,13 +283,13 @@ public class NumassServer extends AbstractNetworkListener { // } // b.append("
\n"); // for (Loader loader : storage.loaders().values()) { -// renderLoader(ctx, b, loader); +// defaultRenderLoader(ctx, b, loader); // } // b.append("
\n"); // b.append("\n"); // } // -// private void renderLoader(Context ctx, StringBuilder b, Loader loader) { +// private void defaultRenderLoader(Context ctx, StringBuilder b, Loader loader) { // String href = "/storage?path="+loader.getFullPath(); // b.append(String.format("

%s (%s)

", href, loader.getName(), loader.getType())); // } diff --git a/numass-storage/numass-server/src/main/java/inr/numass/server/NumassStorageHandler.java b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassStorageHandler.java new file mode 100644 index 00000000..ffd48691 --- /dev/null +++ b/numass-storage/numass-server/src/main/java/inr/numass/server/NumassStorageHandler.java @@ -0,0 +1,89 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package inr.numass.server; + +import freemarker.template.Template; +import hep.dataforge.exceptions.StorageException; +import hep.dataforge.storage.api.ObjectLoader; +import hep.dataforge.storage.api.Storage; +import hep.dataforge.storage.servlet.StorageRatpackHandler; +import hep.dataforge.storage.servlet.Utils; +import java.io.StringWriter; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.LoggerFactory; +import ratpack.handling.Context; + +/** + * + * @author Alexander Nozik + */ +public class NumassStorageHandler extends StorageRatpackHandler { + + public NumassStorageHandler(Storage root) { + super(root); + } + + @Override + @SuppressWarnings("unchecked") + protected void renderObjects(Context ctx, ObjectLoader loader) { + if (NumassRun.RUN_NOTES.equals(loader.getName())) { + try { + ctx.getResponse().contentType("text/html"); + Template template = Utils.freemarkerConfig().getTemplate("NoteLoader.ftl"); + + List notes = getNotes(loader).limit(100).map(note->render(note)).collect(Collectors.toList()); + + Map data = new HashMap(2); + data.put("notes", notes); + + StringWriter writer = new StringWriter(); + template.process(data, writer); + + ctx.render(writer.toString()); + } catch (Exception ex) { + LoggerFactory.getLogger(getClass()).error("Failed to render template", ex); + ctx.render(ex.toString()); + } + } else { + super.renderObjects(ctx, loader); + } + } + + private String render(NumassNote note){ + return String.format("%s: %s %s", note.ref(), note.time(), note.content()); + } + + /** + * Stream of notes in the last to first order + * + * @return + */ + @SuppressWarnings("unchecked") + private Stream getNotes(ObjectLoader noteLoader) { + return noteLoader.fragmentNames().stream().map(new Function() { + @Override + public NumassNote apply(String name) { + try { + return (NumassNote) noteLoader.pull(name); + } catch (StorageException ex) { + return (NumassNote) null; + } + } + }).sorted(new Comparator() { + @Override + public int compare(NumassNote o1, NumassNote o2) { + return -o1.time().compareTo(o2.time()); + } + }); + } + +} diff --git a/numass-storage/numass-server/src/main/java/inr/numass/server/UnknownNumassActionException.java b/numass-storage/numass-server/src/main/java/inr/numass/server/UnknownNumassActionException.java index 16ce6e6a..39c340a3 100644 --- a/numass-storage/numass-server/src/main/java/inr/numass/server/UnknownNumassActionException.java +++ b/numass-storage/numass-server/src/main/java/inr/numass/server/UnknownNumassActionException.java @@ -13,35 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package inr.numass.server; - -/** - * - * @author Alexander Nozik - */ -public class UnknownNumassActionException extends RuntimeException { - - public enum Cause { - IN_DEVELOPMENT, - NOT_SUPPORTED, - DEPRECATED - } - - private final String command; - private final Cause cause; - - /** - * Creates a new instance of UnknownCommandException without - * detail message. - */ - public UnknownNumassActionException(String command, Cause cause) { - this.command = command; - this.cause = cause; - } - - @Override - public String getMessage() { - return String.format("Can't run the action '%s' because it is %s", command, cause.name()); - } - -} +package inr.numass.server; + +/** + * + * @author Alexander Nozik + */ +public class UnknownNumassActionException extends RuntimeException { + + public enum Cause { + IN_DEVELOPMENT, + NOT_SUPPORTED, + DEPRECATED + } + + private final String command; + private final Cause cause; + + /** + * Creates a new instance of UnknownCommandException without + * detail message. + */ + public UnknownNumassActionException(String command, Cause cause) { + this.command = command; + this.cause = cause; + } + + @Override + public String getMessage() { + return String.format("Can't run the action '%s' because it is %s", command, cause.name()); + } + +} diff --git a/numass-storage/numass-server/src/main/resources/templates/NoteLoader.ftl b/numass-storage/numass-server/src/main/resources/templates/NoteLoader.ftl new file mode 100644 index 00000000..b51880a6 --- /dev/null +++ b/numass-storage/numass-server/src/main/resources/templates/NoteLoader.ftl @@ -0,0 +1,14 @@ + + + + Numass run notes + + + + +

Numass experiment run notes:

+ <#list notes as note> +

${note};

+ + + diff --git a/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java b/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java index 6b6a8d1c..4305b1b7 100644 --- a/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java +++ b/numass-storage/src/main/java/inr/numass/storage/NumassDataLoader.java @@ -21,7 +21,6 @@ import hep.dataforge.io.envelopes.DefaultEnvelopeReader; import hep.dataforge.io.envelopes.Envelope; import hep.dataforge.meta.Meta; import hep.dataforge.meta.MetaBuilder; -import hep.dataforge.storage.api.BinaryLoader; import hep.dataforge.storage.api.Storage; import hep.dataforge.storage.loaders.AbstractLoader; import inr.numass.data.NMEvent; @@ -54,13 +53,14 @@ import static org.apache.commons.vfs2.FileType.FOLDER; import org.apache.commons.vfs2.VFS; import org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider; import org.slf4j.LoggerFactory; +import hep.dataforge.storage.api.ObjectLoader; /** * The reader for numass main detector data directory or zip format; * * @author darksnake */ -public class NumassDataLoader extends AbstractLoader implements BinaryLoader, NumassData { +public class NumassDataLoader extends AbstractLoader implements ObjectLoader, NumassData { //FIXME administer resource release /**