Skip to content

Commit c312b02

Browse files
ebiggzebiggz
committed
perf(viewers): use proper server side pagination for viewers tab, increase auto viewer db compaction, add viewer indexes
1 parent 92a8a60 commit c312b02

7 files changed

Lines changed: 197 additions & 106 deletions

File tree

‎src/backend/viewers/viewer-database.ts‎

Lines changed: 62 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import roleHelpers from "../roles/role-helpers";
2424
import teamRolesManager from "../roles/team-roles-manager";
2525
import frontendCommunicator from "../common/frontend-communicator";
2626
import { LoggerCache } from "../logger-cache";
27-
import { commafy, wait } from "../utils";
27+
import { commafy, escapeRegExp, wait } from "../utils";
2828

2929
interface ViewerDbChangePacket {
3030
userId: string;
@@ -56,6 +56,20 @@ interface ViewerPurgeOptions {
5656
};
5757
}
5858

59+
interface ViewersPageRequest {
60+
page: number;
61+
pageSize: number;
62+
sortField?: string;
63+
sortReversed?: boolean;
64+
search?: string;
65+
}
66+
67+
interface ViewersPage {
68+
viewers: FirebotViewer[];
69+
total: number;
70+
totalUnfiltered: number;
71+
}
72+
5973
interface UserDetails {
6074
firebotData: FirebotViewer;
6175
twitchData: Record<string, unknown>;
@@ -70,22 +84,14 @@ class ViewerDatabase extends TypedEmitter<{
7084
private logger = LoggerCache.getLogger("Viewers");
7185

7286
private _db: Datastore<FirebotViewer>;
73-
private _dbCompactionInterval = 30000;
87+
private _dbCompactionInterval = 60 * 60 * 1000; // 1 hour
7488

7589
private cancelRankRecalculation = false;
7690
private _activeViewers: string[] = [];
7791

7892
constructor() {
7993
super();
8094

81-
frontendCommunicator.onAsync("connect-viewer-db", async () => {
82-
if (this.isViewerDBOn() !== true) {
83-
return;
84-
}
85-
await this.connectViewerDatabase();
86-
this.logger.debug("Connecting to viewer database.");
87-
});
88-
8995
frontendCommunicator.onAsync("viewer-db-change", async (data: ViewerDbChangePacket) => {
9096
if (this.isViewerDBOn() !== true) {
9197
return;
@@ -108,11 +114,8 @@ class ViewerDatabase extends TypedEmitter<{
108114
return await this.purgeViewers(options);
109115
});
110116

111-
frontendCommunicator.onAsync("viewer-database:get-all-viewers", async () => {
112-
if (this.isViewerDBOn() !== true) {
113-
return [];
114-
}
115-
return await this.getAllViewers();
117+
frontendCommunicator.onAsync("viewer-database:get-viewers-page", async (request: ViewersPageRequest) => {
118+
return await this.getViewersPage(request);
116119
});
117120

118121
frontendCommunicator.onAsync("create-firebot-viewer-data", async (viewer: BasicViewer) => {
@@ -192,11 +195,15 @@ class ViewerDatabase extends TypedEmitter<{
192195
}, this._dbCompactionInterval);
193196

194197
this.logger.info("Viewer Database Loaded: ", path);
195-
this.emit("viewer-database-loaded");
196-
}
197198

198-
disconnectViewerDatabase(): void {
199-
this._db = null;
199+
try {
200+
await this._db.ensureIndexAsync({ fieldName: "username", unique: false });
201+
await this._db.ensureIndexAsync({ fieldName: "displayName", unique: false });
202+
} catch (error) {
203+
this.logger.error("Error setting up viewer database indexes: ", error);
204+
}
205+
206+
this.emit("viewer-database-loaded");
200207
}
201208

202209
getViewerDb(): Datastore<FirebotViewer> {
@@ -301,6 +308,38 @@ class ViewerDatabase extends TypedEmitter<{
301308
return Object.values(await this._db.findAsync({}));
302309
}
303310

311+
async getViewersPage({ page, pageSize, sortField, sortReversed, search }: ViewersPageRequest): Promise<ViewersPage> {
312+
if (this.isViewerDBOn() !== true) {
313+
return { viewers: [], total: 0, totalUnfiltered: 0 };
314+
}
315+
316+
const query: Record<string, unknown> = {};
317+
if (search != null && search.length > 0) {
318+
const searchRegex = new RegExp(escapeRegExp(search), "i");
319+
query.$or = [
320+
{ username: { $regex: searchRegex } },
321+
{ displayName: { $regex: searchRegex } }
322+
];
323+
}
324+
325+
const sortObj = sortField ? { [sortField]: sortReversed ? -1 : 1 } : {};
326+
327+
try {
328+
const totalUnfiltered = await this._db.countAsync({});
329+
const total = query.$or ? await this._db.countAsync(query) : totalUnfiltered;
330+
331+
const viewers = await this._db.findAsync(query)
332+
.sort(sortObj)
333+
.skip(Math.max(0, (page - 1) * pageSize))
334+
.limit(pageSize);
335+
336+
return { viewers, total, totalUnfiltered };
337+
} catch (error) {
338+
this.logger.error("Error getting viewers page: ", error);
339+
return { viewers: [], total: 0, totalUnfiltered: 0 };
340+
}
341+
}
342+
304343
async getAllUsernames(): Promise<string[]> {
305344
if (this.isViewerDBOn() !== true) {
306345
return [];
@@ -490,7 +529,10 @@ class ViewerDatabase extends TypedEmitter<{
490529

491530
async getPurgeViewers(options: ViewerPurgeOptions): Promise<FirebotViewer[]> {
492531
try {
493-
const bannedUsers = (await TwitchApi.moderation.getBannedUsers()).filter(u => u.expiryDate === null);
532+
let bannedUsers: HelixBan[] = [];
533+
if (options.banned.enabled) {
534+
bannedUsers = (await TwitchApi.moderation.getBannedUsers()).filter(u => u.expiryDate === null);
535+
}
494536
return await this._db.findAsync({ $where: this.getPurgeWherePredicate(options, bannedUsers) });
495537
} catch {
496538
return [];

‎src/backend/viewers/viewer-online-status-manager.ts‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import connectionManager from "../common/connection-manager";
77
import { EventManager } from "../events/event-manager";
88
import twitchChat from "../chat/twitch-chat";
99
import twitchChatterPoll from "../streaming-platforms/twitch/chatter-poll";
10-
import frontendCommunicator from "../common/frontend-communicator";
1110
import { ActiveUserHandler } from "../chat/active-user-handler";
1211
import { LoggerCache } from "../logger-cache";
1312

@@ -28,18 +27,6 @@ class ViewerOnlineStatusManager {
2827
this._updateTimeIntervalId = setInterval(async () => await this.calcAllViewersOnlineMinutes(), 900000);
2928
});
3029

31-
frontendCommunicator.onAsync("disconnect-viewer-db", async () => {
32-
await this.setAllViewersOffline();
33-
34-
// Clear the online time calc interval.
35-
clearInterval(this._updateTimeIntervalId);
36-
clearInterval(this._updateLastSeenIntervalId);
37-
38-
viewerDatabase.disconnectViewerDatabase();
39-
40-
this.logger.debug("Disconnecting from viewer database.");
41-
});
42-
4330
ActiveUserHandler.on("user:online", (user) => {
4431
void this.setChatViewerOnline(user);
4532
});

‎src/gui/app/app-main.js‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,6 @@
124124
updatesService,
125125
commandsService,
126126
integrationService,
127-
viewersService,
128127
chatModerationService,
129128
ttsService,
130129
settingsService,
@@ -167,8 +166,6 @@
167166
//get integrations from backend
168167
integrationService.updateIntegrations();
169168

170-
viewersService.updateViewers();
171-
172169
chatModerationService.loadChatModerationData();
173170

174171
countersService.loadCounters();

‎src/gui/app/controllers/viewers.controller.js‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@
55
angular
66
.module("firebotApp")
77
.controller("viewersController", function(
8-
$route,
9-
$scope,
10-
viewersService,
8+
$route,
9+
$scope,
10+
viewersService,
1111
currencyService,
12-
utilityService,
13-
settingsService,
12+
utilityService,
13+
settingsService,
1414
ngToast
1515
) {
1616
$scope.isViewerDBOn = settingsService.getSetting("ViewerDB");
@@ -90,10 +90,6 @@
9090

9191
$scope.vs = viewersService;
9292

93-
// Update table rows when first visiting the page.
94-
if (viewersService.isViewerDbOn()) {
95-
viewersService.updateViewers();
96-
}
9793

9894
$scope.viewerSearch = "";
9995

‎src/gui/app/directives/misc/sortable-table.js‎

Lines changed: 103 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313
startingSortField: "@",
1414
sortInitiallyReversed: "<",
1515
noDataMessage: "@",
16-
pageSize: "<"
16+
pageSize: "<",
17+
// optional server side fetches, callback should resolve to "{ items, total, totalUnfiltered }"
18+
onFetchPage: "&?",
19+
reloadToken: "<"
1720
},
1821
template: `
1922
<div>
@@ -27,40 +30,63 @@
2730
<span ng-if="header.sortable" style="display:inline-block; width: 11px;">
2831
<i ng-show="$ctrl.isOrderField(header.dataField)" class="fal" ng-class="{'fa-arrow-to-bottom': !$ctrl.order.reverse,'fa-arrow-to-top': $ctrl.order.reverse}"></i>
2932
</span>
30-
</div>
33+
</div>
3134
</th>
3235
</tr>
3336
</thead>
3437
<tbody>
35-
<tr ng-repeat="data in filtered = ($ctrl.tableDataSet | filter:$ctrl.query | orderBy:$ctrl.dynamicOrder:$ctrl.order.reverse) | startFrom:($ctrl.pagination.currentPage-1)*$ctrl.pagination.pageSize | limitTo:$ctrl.pagination.pageSize as visible track by $ctrl.getTrackBy(data, $index)" class="viewer-row" ng-class="{selectable: $ctrl.clickable}"
38+
<tr ng-repeat="data in $ctrl.visibleRows track by $ctrl.getTrackBy(data, $index)" class="viewer-row" ng-class="{selectable: $ctrl.clickable}"
3639
ng-click="$ctrl.clickable && $ctrl.onRowClick({ data: data })">
3740
<td ng-repeat="header in $ctrl.headers track by $index" ng-style="header.cellStyles">
3841
<sortable-table-cell data="data" header="header" cell-index="$index"></sortable-table-cell>
3942
</td>
4043
</tr>
4144
</tbody>
4245
</table>
43-
<div ng-show="$ctrl.tableDataSet.length < 1" class="fb-table-row">
46+
<div ng-show="$ctrl.loading" class="fb-table-row">
47+
<span class="muted">Loading...</span>
48+
</div>
49+
<div ng-show="!$ctrl.loading && $ctrl.totalCount < 1" class="fb-table-row">
4450
<span class="muted">{{$ctrl.noDataMessage ? $ctrl.noDataMessage : "No data available yet"}}</span>
4551
</div>
4652
<div style="display: grid;grid-template-columns: 1fr max-content 1fr;">
4753
<div></div>
4854
<div>
49-
<div ng-show="filtered.length > $ctrl.pagination.pageSize" style="text-align: center;">
50-
<ul uib-pagination total-items="filtered.length" ng-model="$ctrl.pagination.currentPage" items-per-page="$ctrl.pagination.pageSize" class="pagination-sm" max-size="3" boundary-link-numbers="true" rotate="true" style="margin-top:10px;"></ul>
55+
<div ng-show="$ctrl.filteredCount > $ctrl.pagination.pageSize" style="text-align: center;">
56+
<ul uib-pagination total-items="$ctrl.filteredCount" ng-model="$ctrl.pagination.currentPage" items-per-page="$ctrl.pagination.pageSize" class="pagination-sm" max-size="3" boundary-link-numbers="true" rotate="true" style="margin-top:10px;"></ul>
5157
</div>
5258
</div>
5359
<div>
54-
<div ng-hide="$ctrl.tableDataSet.length < 1" class="muted" style="margin-top: 10px;font-size: 11px;text-align: right;">
55-
<span>Showing <strong>{{$ctrl.getRangeMin()}}</strong> - <strong>{{$ctrl.getRangeMax(filtered.length)}}</strong> of <strong>{{$ctrl.tableDataSet.length}}</strong> total</span>
60+
<div ng-hide="$ctrl.totalCount < 1" class="muted" style="margin-top: 10px;font-size: 11px;text-align: right;">
61+
<span>Showing <strong>{{$ctrl.getRangeMin()}}</strong> - <strong>{{$ctrl.getRangeMax($ctrl.filteredCount)}}</strong> of <strong>{{$ctrl.totalCount}}</strong> total</span>
5662
</div>
5763
</div>
5864
</div>
5965
</div>
6066
`,
61-
controller: function() {
67+
controller: function($scope, $filter, $q, $attrs) {
6268
const $ctrl = this;
6369

70+
$ctrl.serverMode = "onFetchPage" in $attrs;
71+
72+
$ctrl.visibleRows = [];
73+
$ctrl.filteredCount = 0;
74+
$ctrl.totalCount = 0;
75+
$ctrl.loading = false;
76+
77+
$ctrl.pagination = {
78+
currentPage: 1,
79+
pageSize: 10
80+
};
81+
82+
$ctrl.order = {
83+
field: '0',
84+
reverse: false
85+
};
86+
87+
let fetchSeq = 0;
88+
let initialized = false;
89+
6490
$ctrl.$onInit = () => {
6591
if ($ctrl.tableDataSet == null) {
6692
$ctrl.tableDataSet = [];
@@ -75,11 +101,76 @@
75101
if ($ctrl.pageSize !== undefined && $ctrl.pageSize !== null) {
76102
$ctrl.pagination.pageSize = $ctrl.pageSize;
77103
}
104+
105+
initialized = true;
106+
107+
// Reset to the first page whenever the search query or sort changes.
108+
$scope.$watchGroup(['$ctrl.query', '$ctrl.order.field', '$ctrl.order.reverse'], (newVals, oldVals) => {
109+
if (newVals === oldVals) {
110+
return;
111+
}
112+
$ctrl.pagination.currentPage = 1;
113+
});
114+
115+
$scope.$watchGroup(
116+
['$ctrl.query', '$ctrl.order.field', '$ctrl.order.reverse', '$ctrl.pagination.currentPage', '$ctrl.pagination.pageSize'],
117+
() => $ctrl.recompute()
118+
);
119+
120+
if (!$ctrl.serverMode) {
121+
$scope.$watchCollection('$ctrl.tableDataSet', () => $ctrl.recompute());
122+
}
78123
};
79124

80-
$ctrl.pagination = {
81-
currentPage: 1,
82-
pageSize: 10
125+
$ctrl.$onChanges = (changes) => {
126+
if ($ctrl.serverMode && initialized && changes.reloadToken && !changes.reloadToken.isFirstChange()) {
127+
$ctrl.recompute();
128+
}
129+
};
130+
131+
$ctrl.recompute = () => {
132+
if ($ctrl.serverMode) {
133+
$ctrl.fetchServerPage();
134+
return;
135+
}
136+
137+
let filtered = $filter('filter')($ctrl.tableDataSet || [], $ctrl.query);
138+
filtered = $filter('orderBy')(filtered, $ctrl.dynamicOrder, $ctrl.order.reverse);
139+
140+
$ctrl.filteredCount = filtered.length;
141+
$ctrl.totalCount = ($ctrl.tableDataSet || []).length;
142+
143+
const start = ($ctrl.pagination.currentPage - 1) * $ctrl.pagination.pageSize;
144+
$ctrl.visibleRows = filtered.slice(start, start + $ctrl.pagination.pageSize);
145+
};
146+
147+
$ctrl.fetchServerPage = () => {
148+
const seq = ++fetchSeq;
149+
$ctrl.loading = true;
150+
151+
$q.when($ctrl.onFetchPage({
152+
params: {
153+
page: $ctrl.pagination.currentPage,
154+
pageSize: $ctrl.pagination.pageSize,
155+
sortField: $ctrl.order.field,
156+
sortReversed: $ctrl.order.reverse,
157+
search: $ctrl.query
158+
}
159+
})).then((result) => {
160+
if (seq !== fetchSeq) {
161+
return;
162+
}
163+
result = result || {};
164+
$ctrl.visibleRows = result.items || [];
165+
$ctrl.filteredCount = result.total != null ? result.total : 0;
166+
$ctrl.totalCount = result.totalUnfiltered != null ? result.totalUnfiltered : $ctrl.filteredCount;
167+
$ctrl.loading = false;
168+
}).catch(() => {
169+
if (seq !== fetchSeq) {
170+
return;
171+
}
172+
$ctrl.loading = false;
173+
});
83174
};
84175

85176
$ctrl.getTrackBy = (data, index) => {
@@ -98,11 +189,6 @@
98189
return max <= filteredLength ? max : filteredLength;
99190
};
100191

101-
$ctrl.order = {
102-
field: '0',
103-
reverse: false
104-
};
105-
106192
$ctrl.isOrderField = function(field) {
107193
return field === $ctrl.order.field;
108194
};

0 commit comments

Comments
 (0)