diff --git a/src/core/hle/ipc_helpers.h b/src/core/hle/ipc_helpers.h
index 026257115b..3c4e45fcd5 100644
--- a/src/core/hle/ipc_helpers.h
+++ b/src/core/hle/ipc_helpers.h
@@ -385,7 +385,7 @@ public:
     T PopRaw();
 
     template <class T>
-    std::shared_ptr<T> PopIpcInterface() {
+    std::weak_ptr<T> PopIpcInterface() {
         ASSERT(context->Session()->IsDomain());
         ASSERT(context->GetDomainMessageHeader().input_object_count > 0);
         return context->GetDomainHandler<T>(Pop<u32>() - 1);
diff --git a/src/core/hle/kernel/hle_ipc.cpp b/src/core/hle/kernel/hle_ipc.cpp
index e19544c54f..38abc4fd16 100644
--- a/src/core/hle/kernel/hle_ipc.cpp
+++ b/src/core/hle/kernel/hle_ipc.cpp
@@ -45,7 +45,7 @@ bool SessionRequestManager::HasSessionRequestHandler(const HLERequestContext& co
             LOG_CRITICAL(IPC, "object_id {} is too big!", object_id);
             return false;
         }
-        return DomainHandler(object_id - 1) != nullptr;
+        return DomainHandler(object_id - 1).lock() != nullptr;
     } else {
         return session_handler != nullptr;
     }
diff --git a/src/core/hle/kernel/hle_ipc.h b/src/core/hle/kernel/hle_ipc.h
index 754b41ff67..670cc741cd 100644
--- a/src/core/hle/kernel/hle_ipc.h
+++ b/src/core/hle/kernel/hle_ipc.h
@@ -94,6 +94,7 @@ protected:
     std::weak_ptr<ServiceThread> service_thread;
 };
 
+using SessionRequestHandlerWeakPtr = std::weak_ptr<SessionRequestHandler>;
 using SessionRequestHandlerPtr = std::shared_ptr<SessionRequestHandler>;
 
 /**
@@ -139,7 +140,7 @@ public:
         }
     }
 
-    SessionRequestHandlerPtr DomainHandler(std::size_t index) const {
+    SessionRequestHandlerWeakPtr DomainHandler(std::size_t index) const {
         ASSERT_MSG(index < DomainHandlerCount(), "Unexpected handler index {}", index);
         return domain_handlers.at(index);
     }
@@ -328,10 +329,10 @@ public:
 
     template <typename T>
     std::shared_ptr<T> GetDomainHandler(std::size_t index) const {
-        return std::static_pointer_cast<T>(manager->DomainHandler(index));
+        return std::static_pointer_cast<T>(manager.lock()->DomainHandler(index).lock());
     }
 
-    void SetSessionRequestManager(std::shared_ptr<SessionRequestManager> manager_) {
+    void SetSessionRequestManager(std::weak_ptr<SessionRequestManager> manager_) {
         manager = std::move(manager_);
     }
 
@@ -374,7 +375,7 @@ private:
     u32 handles_offset{};
     u32 domain_offset{};
 
-    std::shared_ptr<SessionRequestManager> manager;
+    std::weak_ptr<SessionRequestManager> manager;
 
     KernelCore& kernel;
     Core::Memory::Memory& memory;
diff --git a/src/core/hle/kernel/k_port.cpp b/src/core/hle/kernel/k_port.cpp
index a8ba09c4a8..ceb98709f9 100644
--- a/src/core/hle/kernel/k_port.cpp
+++ b/src/core/hle/kernel/k_port.cpp
@@ -57,7 +57,12 @@ ResultCode KPort::EnqueueSession(KServerSession* session) {
     R_UNLESS(state == State::Normal, ResultPortClosed);
 
     server.EnqueueSession(session);
-    server.GetSessionRequestHandler()->ClientConnected(server.AcceptSession());
+
+    if (auto session_ptr = server.GetSessionRequestHandler().lock()) {
+        session_ptr->ClientConnected(server.AcceptSession());
+    } else {
+        UNREACHABLE();
+    }
 
     return ResultSuccess;
 }
diff --git a/src/core/hle/kernel/k_server_port.h b/src/core/hle/kernel/k_server_port.h
index 6302d5e617..2185736be9 100644
--- a/src/core/hle/kernel/k_server_port.h
+++ b/src/core/hle/kernel/k_server_port.h
@@ -30,11 +30,11 @@ public:
 
     /// Whether or not this server port has an HLE handler available.
     bool HasSessionRequestHandler() const {
-        return session_handler != nullptr;
+        return !session_handler.expired();
     }
 
     /// Gets the HLE handler for this port.
-    SessionRequestHandlerPtr GetSessionRequestHandler() const {
+    SessionRequestHandlerWeakPtr GetSessionRequestHandler() const {
         return session_handler;
     }
 
@@ -42,7 +42,7 @@ public:
      * Sets the HLE handler template for the port. ServerSessions crated by connecting to this port
      * will inherit a reference to this handler.
      */
-    void SetSessionHandler(SessionRequestHandlerPtr&& handler) {
+    void SetSessionHandler(SessionRequestHandlerWeakPtr&& handler) {
         session_handler = std::move(handler);
     }
 
@@ -66,7 +66,7 @@ private:
     void CleanupSessions();
 
     SessionList session_list;
-    SessionRequestHandlerPtr session_handler;
+    SessionRequestHandlerWeakPtr session_handler;
     KPort* parent{};
 };
 
diff --git a/src/core/hle/kernel/k_server_session.cpp b/src/core/hle/kernel/k_server_session.cpp
index 4d94eb9cfd..9cfbcbbc90 100644
--- a/src/core/hle/kernel/k_server_session.cpp
+++ b/src/core/hle/kernel/k_server_session.cpp
@@ -98,7 +98,12 @@ ResultCode KServerSession::HandleDomainSyncRequest(Kernel::HLERequestContext& co
             UNREACHABLE();
             return ResultSuccess; // Ignore error if asserts are off
         }
-        return manager->DomainHandler(object_id - 1)->HandleSyncRequest(*this, context);
+        if (auto strong_ptr = manager->DomainHandler(object_id - 1).lock()) {
+            return strong_ptr->HandleSyncRequest(*this, context);
+        } else {
+            UNREACHABLE();
+            return ResultSuccess;
+        }
 
     case IPC::DomainMessageHeader::CommandType::CloseVirtualHandle: {
         LOG_DEBUG(IPC, "CloseVirtualHandle, object_id=0x{:08X}", object_id);
diff --git a/src/core/hle/service/am/am.cpp b/src/core/hle/service/am/am.cpp
index 2f8e215689..420de3c54d 100644
--- a/src/core/hle/service/am/am.cpp
+++ b/src/core/hle/service/am/am.cpp
@@ -980,7 +980,7 @@ private:
         LOG_DEBUG(Service_AM, "called");
 
         IPC::RequestParser rp{ctx};
-        applet->GetBroker().PushNormalDataFromGame(rp.PopIpcInterface<IStorage>());
+        applet->GetBroker().PushNormalDataFromGame(rp.PopIpcInterface<IStorage>().lock());
 
         IPC::ResponseBuilder rb{ctx, 2};
         rb.Push(ResultSuccess);
@@ -1007,7 +1007,7 @@ private:
         LOG_DEBUG(Service_AM, "called");
 
         IPC::RequestParser rp{ctx};
-        applet->GetBroker().PushInteractiveDataFromGame(rp.PopIpcInterface<IStorage>());
+        applet->GetBroker().PushInteractiveDataFromGame(rp.PopIpcInterface<IStorage>().lock());
 
         ASSERT(applet->IsInitialized());
         applet->ExecuteInteractive();