android: improve clipboard, inter-document copy/paste possible

In andorid app, we just supported html or text to be copied between
applications (inside the document the copy/paste is handeled by
the core). The biggest problem with this is that copying between
documents doesn't work or works just as good as our HTML support
is. With this change, we get all the clipboard content after copy
and serialize it to a file into the cache and if we then paste in
another document, we read the content of the file and paste
without any loss of fidelity.

Change-Id: I3501045ff2e815a7e2f8c6ecce5255511c256ca6
Reviewed-on: https://gerrit.libreoffice.org/c/online/+/86833
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>
Tested-by: Michael Meeks <michael.meeks@collabora.com>

4 files changed, 321 insertions, 52 deletions

diff --git a/android/lib/src/main/cpp/androidapp.cpp b/android/lib/src/main/cpp/androidapp.cpp
index 1e5a80686..59079d935 100644
--- a/android/lib/src/main/cpp/androidapp.cpp
+++ b/android/lib/src/main/cpp/androidapp.cpp
@@ -338,7 +338,7 @@ Java_org_libreoffice_androidlib_LOActivity_createLOOLWSD(JNIEnv *env, jobject, j
 extern "C"
 JNIEXPORT void JNICALL
 Java_org_libreoffice_androidlib_LOActivity_saveAs(JNIEnv *env, jobject instance,
-                                                    jstring fileUri_, jstring format_) {
+                                                  jstring fileUri_, jstring format_) {
     const char *fileUri = env->GetStringUTFChars(fileUri_, 0);
     const char *format = env->GetStringUTFChars(format_, 0);
 
@@ -348,6 +348,23 @@ Java_org_libreoffice_androidlib_LOActivity_saveAs(JNIEnv *env, jobject instance,
     env->ReleaseStringUTFChars(format_, format);
 }
 
+extern "C"
+JNIEXPORT void JNICALL
+Java_org_libreoffice_androidlib_LOActivity_postUnoCommand(JNIEnv* pEnv, jobject instance,
+                                                          jstring command, jstring arguments, jboolean bNotifyWhenFinished)
+{
+    const char* pCommand = pEnv->GetStringUTFChars(command, nullptr);
+    const char* pArguments = nullptr;
+    if (arguments != nullptr)
+        pArguments = pEnv->GetStringUTFChars(arguments, nullptr);
+
+    getLOKDocument()->postUnoCommand(pCommand, pArguments, bNotifyWhenFinished);
+
+    pEnv->ReleaseStringUTFChars(command, pCommand);
+    if (arguments != nullptr)
+        pEnv->ReleaseStringUTFChars(arguments, pArguments);
+}
+
 static jstring tojstringAndFree(JNIEnv *env, char *str)
 {
     if (!str)
@@ -357,51 +374,167 @@ static jstring tojstringAndFree(JNIEnv *env, char *str)
     return ret;
 }
 
+const char* copyJavaString(JNIEnv* pEnv, jstring aJavaString)
+{
+    const char* pTemp = pEnv->GetStringUTFChars(aJavaString, nullptr);
+    const char* pClone = strdup(pTemp);
+    pEnv->ReleaseStringUTFChars(aJavaString, pTemp);
+    return pClone;
+}
+
 extern "C"
-JNIEXPORT jobjectArray JNICALL
-Java_org_libreoffice_androidlib_LOActivity_getClipboardContent(JNIEnv *env, jobject instance)
+JNIEXPORT jboolean JNICALL
+Java_org_libreoffice_androidlib_LOActivity_getClipboardContent(JNIEnv *env, jobject instance, jobject lokClipboardData)
 {
-    const char *mimeTypes[] = { "text/plain;charset=utf-8", "text/html", nullptr };
+    const char** mimeTypes = nullptr;
     size_t outCount = 0;
     char  **outMimeTypes = nullptr;
     size_t *outSizes = nullptr;
     char  **outStreams = nullptr;
+    bool bResult = false;
 
-    jobjectArray values = (jobjectArray)env->NewObjectArray(2,env->FindClass("java/lang/String"),env->NewStringUTF(""));
+    jclass jclazz = env->FindClass("java/util/ArrayList");
+    jmethodID methodId_ArrayList_Add = env->GetMethodID(jclazz, "add", "(Ljava/lang/Object;)Z");
+
+    jclass class_LokClipboardEntry = env->FindClass("org/libreoffice/androidlib/lok/LokClipboardEntry");
+    jmethodID methodId_LokClipboardEntry_Constructor = env->GetMethodID(class_LokClipboardEntry, "<init>", "()V");
+    jfieldID fieldId_LokClipboardEntry_Mime = env->GetFieldID(class_LokClipboardEntry , "mime", "Ljava/lang/String;");
+    jfieldID fieldId_LokClipboardEntry_Data = env->GetFieldID(class_LokClipboardEntry, "data", "[B");
+
+    jclass class_LokClipboardData = env->GetObjectClass(lokClipboardData);
+    jfieldID fieldId_LokClipboardData_clipboardEntries = env->GetFieldID(class_LokClipboardData , "clipboardEntries", "Ljava/util/ArrayList;");
 
     if (getLOKDocument()->getClipboard(mimeTypes,
                                        &outCount, &outMimeTypes,
                                        &outSizes, &outStreams))
     {
-        if (outCount != 2)
-            LOG_DBG("clipboard fetch produced wrong results");
-        else
+        // return early
+        if (outCount == 0)
+            return bResult;
+
+        for (size_t i = 0; i < outCount; ++i)
+        {
+            // Create new LokClipboardEntry instance
+            jobject clipboardEntry = env->NewObject(class_LokClipboardEntry, methodId_LokClipboardEntry_Constructor);
+
+            jstring mimeType = tojstringAndFree(env, outMimeTypes[i]);
+            // clipboardEntry.mime= mimeType
+            env->SetObjectField(clipboardEntry, fieldId_LokClipboardEntry_Mime, mimeType);
+
+            size_t aByteArraySize = outSizes[i];
+            jbyteArray aByteArray = env->NewByteArray(aByteArraySize);
+            // Copy char* to bytearray
+            env->SetByteArrayRegion(aByteArray, 0, aByteArraySize, (jbyte*) outStreams[i]);
+            // clipboardEntry.data = aByteArray
+            env->SetObjectField(clipboardEntry, fieldId_LokClipboardEntry_Data, aByteArray);
+
+            // clipboardData.clipboardEntries
+            jobject lokClipboardData_clipboardEntries = env->GetObjectField(lokClipboardData, fieldId_LokClipboardData_clipboardEntries);
+
+            // clipboardEntries.add(clipboardEntry)
+            env->CallBooleanMethod(lokClipboardData_clipboardEntries, methodId_ArrayList_Add, clipboardEntry);
+        }
+        bResult = true;
+    }
+    else
+        LOG_DBG("failed to fetch mime-types");
+
+    const char* mimeTypesHTML[] = { "text/plain;charset=utf-8", "text/html", nullptr };
+
+    if (getLOKDocument()->getClipboard(mimeTypesHTML,
+                                       &outCount, &outMimeTypes,
+                                       &outSizes, &outStreams))
+    {
+        // return early
+        if (outCount == 0)
+            return bResult;
+
+        for (size_t i = 0; i < outCount; ++i)
         {
-            env->SetObjectArrayElement(values,0,tojstringAndFree(env, outStreams[0]));
-            env->SetObjectArrayElement(values,1,tojstringAndFree(env, outStreams[1]));
-            free (outMimeTypes[0]);
-            free (outMimeTypes[1]);
-            free (outMimeTypes);
-            free (outStreams);
+            // Create new LokClipboardEntry instance
+            jobject clipboardEntry = env->NewObject(class_LokClipboardEntry, methodId_LokClipboardEntry_Constructor);
+
+            jstring mimeType = tojstringAndFree(env, outMimeTypes[i]);
+            // clipboardEntry.mime= mimeType
+            env->SetObjectField(clipboardEntry, fieldId_LokClipboardEntry_Mime, mimeType);
+
+            size_t aByteArraySize = outSizes[i];
+            jbyteArray aByteArray = env->NewByteArray(aByteArraySize);
+            // Copy char* to bytearray
+            env->SetByteArrayRegion(aByteArray, 0, aByteArraySize, (jbyte*) outStreams[i]);
+            // clipboardEntry.data = aByteArray
+            env->SetObjectField(clipboardEntry, fieldId_LokClipboardEntry_Data, aByteArray);
+
+            // clipboardData.clipboardEntries
+            jobject lokClipboardData_clipboardEntries = env->GetObjectField(lokClipboardData, fieldId_LokClipboardData_clipboardEntries);
+
+            // clipboardEntries.add(clipboardEntry)
+            env->CallBooleanMethod(lokClipboardData_clipboardEntries, methodId_ArrayList_Add, clipboardEntry);
         }
+        bResult = true;
     }
     else
         LOG_DBG("failed to fetch mime-types");
 
-    return values;
+    return bResult;
 }
 
 extern "C"
 JNIEXPORT void JNICALL
-Java_org_libreoffice_androidlib_LOActivity_paste(JNIEnv *env, jobject instance, jstring mimeType_,
-                                                   jstring data_) {
-    const char *mimeType = env->GetStringUTFChars(mimeType_, 0);
-    const char *data = env->GetStringUTFChars(data_, 0);
-    const size_t nSize = env->GetStringLength(data_);
+Java_org_libreoffice_androidlib_LOActivity_setClipboardContent(JNIEnv *env, jobject instance, jobject lokClipboardData) {
+    jclass class_ArrayList= env->FindClass("java/util/ArrayList");
+    jmethodID methodId_ArrayList_ToArray = env->GetMethodID(class_ArrayList, "toArray", "()[Ljava/lang/Object;");
+
+    jclass class_LokClipboardEntry = env->FindClass("org/libreoffice/androidlib/lok/LokClipboardEntry");
+    jfieldID fieldId_LokClipboardEntry_Mime = env->GetFieldID(class_LokClipboardEntry , "mime", "Ljava/lang/String;");
+    jfieldID fieldId_LokClipboardEntry_Data = env->GetFieldID(class_LokClipboardEntry, "data", "[B");
+
+    jclass class_LokClipboardData = env->GetObjectClass(lokClipboardData);
+    jfieldID fieldId_LokClipboardData_clipboardEntries = env->GetFieldID(class_LokClipboardData , "clipboardEntries", "Ljava/util/ArrayList;");
+
+    jobject lokClipboardData_clipboardEntries = env->GetObjectField(lokClipboardData, fieldId_LokClipboardData_clipboardEntries);
+
+    jobjectArray clipboardEntryArray = (jobjectArray) env->CallObjectMethod(lokClipboardData_clipboardEntries, methodId_ArrayList_ToArray);
+
+    size_t nEntrySize= env->GetArrayLength(clipboardEntryArray);
+
+    if (nEntrySize == 0)
+        return;
+
+    size_t pSizes[nEntrySize];
+    const char* pMimeTypes[nEntrySize];
+    const char* pStreams[nEntrySize];
+
+    for (size_t nEntryIndex = 0; nEntryIndex < nEntrySize; ++nEntryIndex)
+    {
+        jobject clipboardEntry = env->GetObjectArrayElement(clipboardEntryArray, nEntryIndex);
 
-    getLOKDocument()->paste(mimeType, data, nSize);
+        jstring mimetype = (jstring) env->GetObjectField(clipboardEntry, fieldId_LokClipboardEntry_Mime);
+        jbyteArray data = (jbyteArray) env->GetObjectField(clipboardEntry, fieldId_LokClipboardEntry_Data);
 
-    env->ReleaseStringUTFChars(mimeType_, mimeType);
-    env->ReleaseStringUTFChars(data_, data);
+        pMimeTypes[nEntryIndex] = copyJavaString(env, mimetype);
+
+        size_t dataArrayLength = env->GetArrayLength(data);
+        char* dataArray = new char[dataArrayLength];
+        env->GetByteArrayRegion(data, 0, dataArrayLength, reinterpret_cast<jbyte*>(dataArray));
+
+        pSizes[nEntryIndex] = dataArrayLength;
+        pStreams[nEntryIndex] = dataArray;
+    }
+
+    getLOKDocument()->setClipboard(nEntrySize, pMimeTypes, pSizes, pStreams);
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_org_libreoffice_androidlib_LOActivity_paste(JNIEnv *env, jobject instance, jstring inMimeType, jbyteArray inData) {
+    const char* mimeType = env->GetStringUTFChars(inMimeType, nullptr);
+
+    size_t dataArrayLength = env->GetArrayLength(inData);
+    char* dataArray = new char[dataArrayLength];
+    env->GetByteArrayRegion(inData, 0, dataArrayLength, reinterpret_cast<jbyte*>(dataArray));
+    getLOKDocument()->paste(mimeType, dataArray, dataArrayLength);
+    env->ReleaseStringUTFChars(inMimeType, mimeType);
 }
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab cinoptions=b1,g0,N-s cinkeys+=0=break: */
diff --git a/android/lib/src/main/java/org/libreoffice/androidlib/LOActivity.java b/android/lib/src/main/java/org/libreoffice/androidlib/LOActivity.java
index 030f35357..7698c9368 100644
--- a/android/lib/src/main/java/org/libreoffice/androidlib/LOActivity.java
+++ b/android/lib/src/main/java/org/libreoffice/androidlib/LOActivity.java
@@ -22,7 +22,6 @@ import android.content.pm.PackageManager;
 import android.content.res.AssetFileDescriptor;
 import android.content.res.AssetManager;
 import android.net.Uri;
-import android.os.AsyncTask;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Handler;
@@ -53,6 +52,7 @@ import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.nio.channels.ReadableByteChannel;
+import java.nio.charset.Charset;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
@@ -62,6 +62,9 @@ import androidx.core.app.ActivityCompat;
 import androidx.core.content.ContextCompat;
 import androidx.localbroadcastmanager.content.LocalBroadcastManager;
 
+import org.libreoffice.androidlib.lok.LokClipboardData;
+import org.libreoffice.androidlib.lok.LokClipboardEntry;
+
 public class LOActivity extends AppCompatActivity {
     final static String TAG = "LOActivity";
 
@@ -73,7 +76,8 @@ public class LOActivity extends AppCompatActivity {
     private static final String KEY_DOCUMENT_URI = "documentUri";
     private static final String KEY_IS_EDITABLE = "isEditable";
     private static final String KEY_INTENT_URI = "intentUri";
-
+    private static final String CLIPBOARD_FILE_PATH = "LibreofficeClipboardFile.data";
+    private static final String CLIPBOARD_LOOL_SIGNATURE = "lool-clip-magic-4a22437e49a8-";
     private File mTempFile = null;
 
     private int providerId;
@@ -669,13 +673,17 @@ public class LOActivity extends AppCompatActivity {
 
     public native void saveAs(String fileUri, String format);
 
-    public native String[] getClipboardContent();
+    public native boolean getClipboardContent(LokClipboardData aData);
+
+    public native void setClipboardContent(LokClipboardData aData);
+
+    public native void paste(String mimeType, byte[] data);
 
-    public native void paste(String mimeType, String data);
+    public native void postUnoCommand(String command, String arguments, boolean bNotifyWhenFinished);
 
     /// Returns a magic that specifies this application - and this document.
     private final String getClipboardMagic() {
-        return "lool-clip-magic-4a22437e49a8-" + Long.toString(loadDocumentMillis);
+        return CLIPBOARD_LOOL_SIGNATURE + Long.toString(loadDocumentMillis);
     }
 
     /// Needs to be executed after the .uno:Copy / Paste has executed
@@ -685,29 +693,38 @@ public class LOActivity extends AppCompatActivity {
         nativeHandler.post(new Runnable() {
                 @Override
                 public void run() {
-                    // text[0], html[1]
-                    String[] clipStrings = LOActivity.this.getClipboardContent();
-                    if (clipStrings.length < 2 ||
-                        (clipStrings[0].length() == 0 && clipStrings[1].length() == 0))
+                    File clipboardFile = new File(getApplicationContext().getCacheDir(), CLIPBOARD_FILE_PATH);
+                    if (clipboardFile.exists())
+                        clipboardFile.delete();
+
+                    LokClipboardData clipboardData = new LokClipboardData();
+                    if (!LOActivity.this.getClipboardContent(clipboardData))
                         Log.e(TAG, "no clipboard to copy");
                     else
                     {
-                        String text = clipStrings[0];
-                        String html = clipStrings[1];
-
-                        int idx = html.indexOf("<meta name=\"generator\" content=\"");
-                        if (idx < 0)
-                            idx = html.indexOf("<meta http-equiv=\"content-type\" content=\"text/html;");
-                        if (idx >= 0) { // inject our magic
-                            StringBuffer newHtml = new StringBuffer(html);
-                            newHtml.insert(idx, "<meta name=\"origin\" content=\"" + getClipboardMagic() + "\"/>\n");
-                            html = newHtml.toString();
-                        }
+                        clipboardData.writeToFile(clipboardFile);
+
+                        String text = clipboardData.getText();
+                        String html = clipboardData.getHtml();
 
-                        if (text == null || text.length() == 0)
-                            Log.i(TAG, "set text to clipoard with: text '" + text + "' and html '" + html + "'");
-                        clipData = ClipData.newHtmlText(ClipDescription.MIMETYPE_TEXT_HTML, text, html);
-                        clipboardManager.setPrimaryClip(clipData);
+                        if (html != null) {
+                            int idx = html.indexOf("<meta name=\"generator\" content=\"");
+
+                            if (idx < 0)
+                                idx = html.indexOf("<meta http-equiv=\"content-type\" content=\"text/html;");
+
+                            if (idx >= 0) { // inject our magic
+                                StringBuffer newHtml = new StringBuffer(html);
+                                newHtml.insert(idx, "<meta name=\"origin\" content=\"" + getClipboardMagic() + "\"/>\n");
+                                html = newHtml.toString();
+                            }
+
+                            if (text == null || text.length() == 0)
+                                Log.i(TAG, "set text to clipoard with: text '" + text + "' and html '" + html + "'");
+
+                            clipData = ClipData.newHtmlText(ClipDescription.MIMETYPE_TEXT_HTML, text, html);
+                            clipboardManager.setPrimaryClip(clipData);
+                        }
                     }
                 }
             });
@@ -721,15 +738,41 @@ public class LOActivity extends AppCompatActivity {
         if (clipDesc != null) {
             if (clipDesc.hasMimeType(ClipDescription.MIMETYPE_TEXT_HTML)) {
                 final String html = clipData.getItemAt(0).getHtmlText();
-                if (html.contains(getClipboardMagic())) {
-                    Log.i(TAG, "clipboard comes from us: short circuit it " + html);
-                    return true;
+                // Check if the clipboard content was made with the app
+                if (html.contains(CLIPBOARD_LOOL_SIGNATURE)) {
+                    // Check if the clipboard content is from the same app instance
+                    if (html.contains(getClipboardMagic())) {
+                        Log.i(TAG, "clipboard comes from us - same instance: short circuit it " + html);
+                        return true;
+                    } else {
+                        Log.i(TAG, "clipboard comes from us - other instance: paste from clipboard file");
+                        nativeHandler.post(new Runnable() {
+                            @Override
+                            public void run() {
+                                File clipboardFile = new File(getApplicationContext().getCacheDir(), CLIPBOARD_FILE_PATH);
+                                LokClipboardData clipboardData = null;
+                                if (clipboardFile.exists()) {
+                                    clipboardData = new LokClipboardData();
+                                    clipboardData.loadFromFile(clipboardFile);
+                                }
+                                if (clipboardData != null) {
+                                    LOActivity.this.setClipboardContent(clipboardData);
+                                    LOActivity.this.postUnoCommand(".uno:Paste", null, false);
+                                } else {
+                                    // Couldn't get data from the clipboard file, but we can still paste html
+                                    byte[] htmlByteArray = html.getBytes(Charset.forName("UTF-8"));
+                                    LOActivity.this.paste("text/html", htmlByteArray);
+                                }
+                            }
+                        });
+                    }
                 } else {
                     Log.i(TAG, "foreign html '" + html + "'");
                     nativeHandler.post(new Runnable() {
                         @Override
                         public void run() {
-                            LOActivity.this.paste("text/html", html);
+                            byte[] htmlByteArray = html.getBytes(Charset.forName("UTF-8"));
+                            LOActivity.this.paste("text/html", htmlByteArray);
                         }
                     });
                 }
@@ -739,7 +782,9 @@ public class LOActivity extends AppCompatActivity {
                 nativeHandler.post(new Runnable() {
                     @Override
                     public void run() {
-                        LOActivity.this.paste("text/plain;charset=utf-16", clipItem.getText().toString());
+                        String text = clipItem.getText().toString();
+                        byte[] textByteArray = text.getBytes(Charset.forName("UTF-16"));
+                        LOActivity.this.paste("text/plain;charset=utf-16", textByteArray);
                     }
                 });
             }
diff --git a/android/lib/src/main/java/org/libreoffice/androidlib/lok/LokClipboardData.java b/android/lib/src/main/java/org/libreoffice/androidlib/lok/LokClipboardData.java
new file mode 100644
index 000000000..3d98b5dfe
--- /dev/null
+++ b/android/lib/src/main/java/org/libreoffice/androidlib/lok/LokClipboardData.java
@@ -0,0 +1,85 @@
+package org.libreoffice.androidlib.lok;
+
+import android.util.Base64;
+import android.util.JsonReader;
+import android.util.JsonWriter;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class LokClipboardData {
+    public ArrayList<LokClipboardEntry> clipboardEntries = new ArrayList<LokClipboardEntry>();
+
+    public String getText() {
+        for (LokClipboardEntry aEntry : clipboardEntries) {
+            if (aEntry.mime.startsWith("text/plain")) { // text/plain;charset=utf-8
+                return new String(aEntry.data, StandardCharsets.UTF_8);
+            }
+        }
+        return null;
+    }
+
+    public String getHtml() {
+        for (LokClipboardEntry aEntry : clipboardEntries) {
+            if (aEntry.mime.startsWith("text/html")){
+                return new String(aEntry.data, StandardCharsets.UTF_8);
+            }
+        }
+        return null;
+    }
+
+    public boolean writeToFile(File file) {
+        try {
+            FileWriter fileWriter = new FileWriter(file.getAbsoluteFile());
+            JsonWriter writer = new JsonWriter(fileWriter);
+            writer.setIndent(" ");
+            writer.beginObject();
+            for (LokClipboardEntry entry : clipboardEntries) {
+                writer.name(entry.mime);
+                writer.value(Base64.encodeToString(entry.data, Base64.DEFAULT));
+            }
+            writer.endObject();
+            writer.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+            return false;
+        }
+        return true;
+    }
+
+    public boolean loadFromFile(File file) {
+        try {
+            clipboardEntries.clear();
+
+            FileReader fileReader= new FileReader(file.getAbsoluteFile());
+            JsonReader reader = new JsonReader(fileReader);
+            reader.beginObject();
+            while (reader.hasNext()) {
+                LokClipboardEntry entry = new LokClipboardEntry();
+                entry.mime = reader.nextName();
+                entry.data = Base64.decode(reader.nextString(), Base64.DEFAULT);
+                clipboardEntries.add(entry);
+            }
+            reader.endObject();
+        } catch (IOException e) {
+            e.printStackTrace();
+            return false;
+        }
+        return true;
+    }
+
+    public LokClipboardEntry getBest() {
+        if (!clipboardEntries.isEmpty()) {
+            return clipboardEntries.get(0);
+        }
+        return null;
+    }
+}
+
diff --git a/android/lib/src/main/java/org/libreoffice/androidlib/lok/LokClipboardEntry.java b/android/lib/src/main/java/org/libreoffice/androidlib/lok/LokClipboardEntry.java
new file mode 100644
index 000000000..3593f0394
--- /dev/null
+++ b/android/lib/src/main/java/org/libreoffice/androidlib/lok/LokClipboardEntry.java
@@ -0,0 +1,6 @@
+package org.libreoffice.androidlib.lok;
+
+public class LokClipboardEntry {
+    public String mime;
+    public byte[] data;
+}