From add6670eb9b56e3b49042ce70f7cf9f51a28bcc9 Mon Sep 17 00:00:00 2001
From: Stanislaw Adaszewski <stanislaw.adaszewski@roche.com>
Date: Sat, 9 May 2020 07:58:07 +0200
Subject: [PATCH 1/2] Added weight_variable_glorot

---
 .gitignore                  |  2 ++
 decagon_pytorch/convolve.py |  0
 decagon_pytorch/weight.py   | 13 +++++++++++++
 3 files changed, 15 insertions(+)
 create mode 100644 .gitignore
 create mode 100755 decagon_pytorch/convolve.py
 create mode 100755 decagon_pytorch/weight.py

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..c20c2ab
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+__pycache__
+
diff --git a/decagon_pytorch/convolve.py b/decagon_pytorch/convolve.py
new file mode 100755
index 0000000..e69de29
diff --git a/decagon_pytorch/weight.py b/decagon_pytorch/weight.py
new file mode 100755
index 0000000..5321660
--- /dev/null
+++ b/decagon_pytorch/weight.py
@@ -0,0 +1,13 @@
+import torch
+import numpy as np
+
+
+def weight_variable_glorot(input_dim, output_dim):
+    """Create a weight variable with Glorot & Bengio (AISTATS 2010)
+    initialization.
+    """
+    init_range = np.sqrt(6.0 / (input_dim + output_dim))
+    initial = -init_range + 2 * init_range * \
+        torch.rand(( input_dim, output_dim ), dtype=torch.float32)
+    initial = initial.requires_grad_(True)
+    return initial

From 142bb3aef34d707b83c46edad07925b4c2e52d1a Mon Sep 17 00:00:00 2001
From: Stanislaw Adaszewski <stanislaw.adaszewski@roche.com>
Date: Sat, 9 May 2020 08:44:52 +0200
Subject: [PATCH 2/2] Added dropout_sparse()

---
 decagon_pytorch/__init__.py               |  3 +++
 decagon_pytorch/dropout.py                | 18 ++++++++++++++++++
 decagon_pytorch/model.py                  |  0
 decagon_pytorch/{weight.py => weights.py} |  2 +-
 4 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 decagon_pytorch/__init__.py
 create mode 100644 decagon_pytorch/dropout.py
 create mode 100644 decagon_pytorch/model.py
 rename decagon_pytorch/{weight.py => weights.py} (85%)
 mode change 100755 => 100644

diff --git a/decagon_pytorch/__init__.py b/decagon_pytorch/__init__.py
new file mode 100644
index 0000000..f628a28
--- /dev/null
+++ b/decagon_pytorch/__init__.py
@@ -0,0 +1,3 @@
+from .weights import *
+from .convolve import *
+from .model import *
diff --git a/decagon_pytorch/dropout.py b/decagon_pytorch/dropout.py
new file mode 100644
index 0000000..3162572
--- /dev/null
+++ b/decagon_pytorch/dropout.py
@@ -0,0 +1,18 @@
+import torch
+
+
+def dropout_sparse(x, keep_prob):
+    """Dropout for sparse tensors.
+    """
+    x = x.coalesce()
+    i = x._indices()
+    v = x._values()
+    size = x.size()
+
+    n = keep_prob + torch.rand(len(v))
+    n = torch.floor(n).to(torch.bool)
+    i = i[:,n]
+    v = v[n]
+    x = torch.sparse_coo_tensor(i, v, size=size)
+
+    return x * (1./keep_prob)
diff --git a/decagon_pytorch/model.py b/decagon_pytorch/model.py
new file mode 100644
index 0000000..e69de29
diff --git a/decagon_pytorch/weight.py b/decagon_pytorch/weights.py
old mode 100755
new mode 100644
similarity index 85%
rename from decagon_pytorch/weight.py
rename to decagon_pytorch/weights.py
index 5321660..305a70f
--- a/decagon_pytorch/weight.py
+++ b/decagon_pytorch/weights.py
@@ -2,7 +2,7 @@ import torch
 import numpy as np
 
 
-def weight_variable_glorot(input_dim, output_dim):
+def init_glorot(input_dim, output_dim):
     """Create a weight variable with Glorot & Bengio (AISTATS 2010)
     initialization.
     """