From 115fa7af03430ef05ba2f71b3096f4b3772e2598 Mon Sep 17 00:00:00 2001
From: hnu202409060624 <2804411502@qq.com>
Date: Mon, 30 Dec 2024 18:26:00 +0800
Subject: [PATCH] ADD file via upload

---
 Feed_Forward.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 Feed_Forward.py

diff --git a/Feed_Forward.py b/Feed_Forward.py
new file mode 100644
index 0000000..d1c2b41
--- /dev/null
+++ b/Feed_Forward.py
@@ -0,0 +1,16 @@
+from torch import nn
+import config
+class PoswiseFeedForwardNet(nn.Module):
+    def __init__(self):
+        super(PoswiseFeedForwardNet, self).__init__()
+        self.fc = nn.Sequential(
+            nn.Linear(config.input_dim, config.d_ff1, bias=config.bias),
+            nn.ReLU(),
+            nn.Linear(config.d_ff1, config.input_dim, bias=config.bias))
+
+    def forward(self, inputs):  # inputs: [batch_size, seq_len, d_model]
+        residual = inputs
+
+        output = self.fc(inputs)
+
+        return nn.LayerNorm(config.input_dim).to('cuda:0')(output + residual)  # [batch_size, seq_len, d_model]