We have 3 pop’s connected with same db , we are having issues on a single node where clients are getting keep disconnecting. Here are some server logs for that pop.
[ERROR] Error in transaction runner thread
[ERROR] Settings check failed
[ERROR] Error in host keep alive update
[ERROR] Error in tasks run thread
[ERROR] Failed to push vpn output
That appears to be issues connecting to the database. The full error messages would be needed for more information. The date and time on the Pritunl hosts is used to allow hosts to monitor the state of other hosts. It is important that the time is accurate across all hosts, this can cause servers to unexpectedly restart if it is not correct.
[EU_POP][2023-04-04 01:25:16,720][ERROR] Error in transaction runner thread
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/transaction.py”, line 21, in _check_thread
for doc in collection.find(spec).sort(‘priority’):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1238, in next
if len(self.__data) or self._refresh():
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1130, in _refresh
self.__session = self.__collection.database.client._ensure_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
[EU_POP][2023-04-04 01:25:18,221][ERROR] Error in tasks run thread
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/task.py”, line 56, in run_thread
tsk = task_cls(id=run_id, upsert=True)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/mongo/object.py”, line 27, in new
mongo_object.load(doc=doc, spec=spec, fields=fields)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/mongo/object.py”, line 74, in load
doc = self.collection.find_one(spec, list(fields))
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 1328, in find_one
for result in cursor.limit(-1):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1238, in next
if len(self.__data) or self._refresh():
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1130, in _refresh
self.__session = self.__collection.database.client._ensure_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
[EU_POP][2023-04-04 01:25:20,222][ERROR] Settings check failed
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/settings.py”, line 13, in _check
settings.reload_mongo()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/settings/settings.py”, line 109, in reload_mongo
for doc in self.collection.find():
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1238, in next
if len(self.__data) or self._refresh():
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1130, in _refresh
self.__session = self.__collection.database.client._ensure_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
[EU_POP][2023-04-04 01:25:22,723][ERROR] Failed to push vpn output
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/server/instance.py”, line 912, in _openvpn_stdout
self.server.output.push_output(line.decode())
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/server/output.py”, line 60, in push_output
self.collection.insert({
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 3339, in insert
return self._insert(doc_or_docs, not continue_on_error,
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 620, in _insert
return self._insert_one(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 609, in _insert_one
self.__database.client._retryable_write(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1551, in _retryable_write
with self._tmp_session(session) as s:
File “/usr/lib/python3.8/contextlib.py”, line 113, in enter
return next(self.gen)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1948, in _tmp_session
s = self._ensure_session(session)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
server_id = “6406043fc0dee1812bc4323f”
[EU_POP][2023-04-04 01:25:38,230][ERROR] Error in host keep alive update
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/host.py”, line 123, in _keep_alive_thread
settings.local.host.collection.update({
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 3371, in update
return self._update_retryable(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 877, in _update_retryable
return self.__database.client._retryable_write(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1551, in _retryable_write
with self._tmp_session(session) as s:
File “/usr/lib/python3.8/contextlib.py”, line 113, in enter
return next(self.gen)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1948, in _tmp_session
s = self._ensure_session(session)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
host_id = “2f37d57a9fac424f92a15a785c3be17a”
host_name = “EU_POP”
[EU_POP][2023-04-04 01:25:45,733][ERROR] Error in task check thread
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/task.py”, line 78, in check_thread
for task_item in task.iter_tasks(spec):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/task.py”, line 110, in iter_tasks
for doc in Task.collection.find(spec or {}):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1238, in next
if len(self.__data) or self._refresh():
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1130, in _refresh
self.__session = self.__collection.database.client._ensure_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
[EU_POP][2023-04-04 01:25:46,235][ERROR] Error in log runner thread
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/logger.py”, line 32, in _logger_runner_thread
collection.insert(msg_docs)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 3339, in insert
return self._insert(doc_or_docs, not continue_on_error,
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 656, in _insert
blk.execute(write_concern, session=session)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/bulk.py”, line 533, in execute
return self.execute_command(generator, write_concern, session)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/bulk.py”, line 361, in execute_command
with client._tmp_session(session) as s:
File “/usr/lib/python3.8/contextlib.py”, line 113, in enter
return next(self.gen)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1948, in _tmp_session
s = self._ensure_session(session)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
[EU_POP][2023-04-04 01:25:47,234][ERROR] Error in transaction runner thread
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/transaction.py”, line 21, in _check_thread
for doc in collection.find(spec).sort(‘priority’):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1238, in next
if len(self.__data) or self._refresh():
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1130, in _refresh
self.__session = self.__collection.database.client._ensure_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>
[EU_POP][2023-04-04 01:25:48,735][ERROR] Error in tasks run thread
Traceback (most recent call last):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/runners/task.py”, line 56, in run_thread
tsk = task_cls(id=run_id, upsert=True)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/mongo/object.py”, line 27, in new
mongo_object.load(doc=doc, spec=spec, fields=fields)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pritunl/mongo/object.py”, line 74, in load
doc = self.collection.find_one(spec, list(fields))
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/collection.py”, line 1328, in find_one
for result in cursor.limit(-1):
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1238, in next
if len(self.__data) or self._refresh():
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/cursor.py”, line 1130, in _refresh
self.__session = self.__collection.database.client._ensure_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1935, in _ensure_session
return self.__start_session(True, causal_consistency=False)
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1883, in __start_session
server_session = self._get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/mongo_client.py”, line 1921, in _get_server_session
return self._topology.get_server_session()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 520, in get_server_session
session_timeout = self._check_session_support()
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 499, in _check_session_support
self._select_servers_loop(
File “/usr/lib/pritunl/lib/python3.8/site-packages/pymongo/topology.py”, line 218, in _select_servers_loop
raise ServerSelectionTimeoutError(
pymongo.errors.ServerSelectionTimeoutError: 10.10.0.102:27017: timed out, Timeout: 30.0s, Topology Description: <TopologyDescription id: 642aaa99d4f6874dbb1278d4, topology_type: Single, servers: [<ServerDescription (‘10.10.0.102’, 27017) server_type: Unknown, rtt: None, error=NetworkTimeout(‘10.10.0.102:27017: timed out’)>]>