优化云端录像定时清理

pull/1242/head
648540858 2023-10-19 17:52:19 +08:00
parent 9b036fb71d
commit daac0010b1
13 changed files with 340 additions and 74 deletions

View File

@ -5,7 +5,7 @@ alter table wvp_platform
add auto_push_channel bool default false
alter table wvp_stream_proxy
add stream_key varying(255)
add stream_key character varying(255)
create table wvp_cloud_record (
id serial primary key,
@ -18,8 +18,17 @@ create table wvp_cloud_record (
file_name character varying(255),
folder character varying(255),
file_path character varying(255),
collect_type character varying(255),
collect bool default false,
reserve bool default false,
file_size integer,
time_len integer,
constraint uk_stream_push_app_stream_path unique (app, stream, file_path)
);
alter table wvp_media_server
add record_path character varying(255);
alter table wvp_media_server
add record_date integer default 7;

View File

@ -164,6 +164,8 @@ create table wvp_media_server (
create_time character varying(50),
update_time character varying(50),
hook_alive_interval integer,
record_path character varying(255),
record_date integer default 7,
constraint uk_media_server_unique_ip_http_port unique (ip, http_port)
);
@ -267,20 +269,21 @@ create table wvp_stream_push (
constraint uk_stream_push_app_stream unique (app, stream)
);
create table wvp_cloud_record (
id serial primary key,
app character varying(255),
stream character varying(255),
call_id character varying(255),
start_time integer,
end_time integer,
media_server_id character varying(50),
file_name character varying(255),
folder character varying(255),
file_path character varying(255),
collect_type character varying(255),
file_size integer,
time_len integer,
constraint uk_stream_push_app_stream_path unique (app, stream, file_path)
id serial primary key,
app character varying(255),
stream character varying(255),
call_id character varying(255),
start_time integer,
end_time integer,
media_server_id character varying(50),
file_name character varying(255),
folder character varying(255),
file_path character varying(255),
collect bool default false,
reserve bool default false,
file_size integer,
time_len integer,
constraint uk_stream_push_app_stream_path unique (app, stream, file_path)
);
create table wvp_user (

View File

@ -0,0 +1,73 @@
package com.genersoft.iot.vmp.conf;
import com.alibaba.fastjson2.JSONObject;
import com.genersoft.iot.vmp.media.zlm.AssistRESTfulUtils;
import com.genersoft.iot.vmp.media.zlm.dto.MediaServerItem;
import com.genersoft.iot.vmp.service.IMediaServerService;
import com.genersoft.iot.vmp.storager.dao.CloudRecordServiceMapper;
import com.genersoft.iot.vmp.vmanager.cloudRecord.CloudRecordController;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
/**
*
*/
@Component
public class CloudRecordTimer {
private final static Logger logger = LoggerFactory.getLogger(CloudRecordTimer.class);
@Autowired
private IMediaServerService mediaServerService;
@Autowired
private CloudRecordServiceMapper cloudRecordServiceMapper;
@Autowired
private AssistRESTfulUtils assistRESTfulUtils;
/**
*
*/
@Scheduled(cron = "0 0 0 * * ?") //每天的0点执行
public void execute(){
// 获取配置了assist的流媒体节点
List<MediaServerItem> mediaServerItemList = mediaServerService.getAllWithAssistPort();
if (mediaServerItemList.isEmpty()) {
return;
}
long result = 0;
for (MediaServerItem mediaServerItem : mediaServerItemList) {
Calendar lastCalendar = Calendar.getInstance();
if (mediaServerItem.getRecordDate() > 0) {
lastCalendar.setTime(new Date());
// 获取保存的最后截至日期,因为每个节点都有一个日期,也就是支持每个节点设置不同的保存日期,
lastCalendar.add(Calendar.DAY_OF_MONTH, -mediaServerItem.getRecordDate());
Long lastDate = lastCalendar.getTimeInMillis();
// 获取到截至日期之前的录像文件列表,文件列表满足未被收藏和保持的。这两个字段目前共能一致,
// 为我自己业务系统相关的代码大家使用的时候直接使用收藏collect这一个类型即可
List<String> filePathList = cloudRecordServiceMapper.queryRecordFilePathListForDelete(lastDate, mediaServerItem.getId());
if (filePathList.isEmpty()) {
continue;
}
// 先调用assist删除磁盘文件删除成功后再删除数据库记录
JSONObject jsonObject = assistRESTfulUtils.deleteFiles(mediaServerItem, filePathList);
if (jsonObject != null && jsonObject.getInteger("code") == 0 && jsonObject.getInteger("data") > 0) {
result += jsonObject.getInteger("data");
cloudRecordServiceMapper.deleteByFileList(filePathList, mediaServerItem.getId());
}
}
}
logger.info("[录像文件定时清理] 共清理{}个过期录像文件", result);
}
}

View File

@ -29,8 +29,6 @@ public class AssistRESTfulUtils {
private OkHttpClient client;
public interface RequestCallback{
void run(JSONObject response);
}
@ -271,4 +269,13 @@ public class AssistRESTfulUtils {
return sendGet(mediaServerItem, "api/record/file/download/task/list", param, null);
}
public JSONObject addCollect(MediaServerItem mediaServerItem, JSONObject jsonObject) {
return sendPost(mediaServerItem, "api/record/file/collection/add", jsonObject, null, 30);
}
public JSONObject deleteFiles(MediaServerItem mediaServerItem, List<String> filePathList) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("filePathList", filePathList);
return sendPost(mediaServerItem, "api/record/file/delete", jsonObject, null, 15*60);
}
}

View File

@ -80,8 +80,13 @@ public class MediaServerItem{
@Schema(description = "是否是默认ZLM")
private boolean defaultServer;
@Schema(description = "当前使用到的端口")
private int currentPort;
@Schema(description = "录像存储路径")
private String recordPath;
@Schema(description = "录像存储时长")
private int recordDate;
public MediaServerItem() {
@ -269,14 +274,6 @@ public class MediaServerItem{
this.updateTime = updateTime;
}
public int getCurrentPort() {
return currentPort;
}
public void setCurrentPort(int currentPort) {
this.currentPort = currentPort;
}
public boolean isStatus() {
return status;
}
@ -308,4 +305,20 @@ public class MediaServerItem{
public void setSendRtpPortRange(String sendRtpPortRange) {
this.sendRtpPortRange = sendRtpPortRange;
}
public String getRecordPath() {
return recordPath;
}
public void setRecordPath(String recordPath) {
this.recordPath = recordPath;
}
public int getRecordDate() {
return recordDate;
}
public void setRecordDate(int recordDate) {
this.recordDate = recordDate;
}
}

View File

@ -1,7 +1,6 @@
package com.genersoft.iot.vmp.service;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.genersoft.iot.vmp.media.zlm.dto.MediaServerItem;
import com.genersoft.iot.vmp.media.zlm.dto.hook.OnRecordMp4HookParam;
import com.genersoft.iot.vmp.service.bean.CloudRecordItem;
@ -40,4 +39,15 @@ public interface ICloudRecordService {
*
*/
JSONArray queryTask(String taskId, String mediaServerId, Boolean isEnd);
/**
*
*/
void changeCollect(String type, boolean result, String app, String stream, String mediaServerId, String startTime, String endTime, String callId, String collectType);
/**
*
*/
void changeCollectById(Integer recordId, String collectType, boolean result);
}

View File

@ -93,4 +93,6 @@ public interface IMediaServerService {
*/
MediaServerLoad getLoad(MediaServerItem mediaServerItem);
List<MediaServerItem> getAllWithAssistPort();
}

View File

@ -57,9 +57,14 @@ public class CloudRecordItem {
private String folder;
/**
*
*
*/
private String collectType;
private Boolean collect;
/**
*
*/
private Boolean reserve;
/**
*
@ -182,11 +187,19 @@ public class CloudRecordItem {
this.timeLen = timeLen;
}
public String getCollectType() {
return collectType;
public Boolean getCollect() {
return collect;
}
public void setCollectType(String collectType) {
this.collectType = collectType;
public void setCollect(Boolean collect) {
this.collect = collect;
}
public Boolean getReserve() {
return reserve;
}
public void setReserve(Boolean reserve) {
this.reserve = reserve;
}
}

View File

@ -3,8 +3,6 @@ package com.genersoft.iot.vmp.service.impl;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.genersoft.iot.vmp.conf.exception.ControllerException;
import com.genersoft.iot.vmp.gb28181.bean.GbStream;
import com.genersoft.iot.vmp.gb28181.bean.SsrcTransaction;
import com.genersoft.iot.vmp.gb28181.session.VideoStreamSessionManager;
import com.genersoft.iot.vmp.media.zlm.AssistRESTfulUtils;
import com.genersoft.iot.vmp.media.zlm.dto.MediaServerItem;
@ -19,14 +17,13 @@ import com.genersoft.iot.vmp.utils.DateUtil;
import com.genersoft.iot.vmp.vmanager.bean.ErrorCode;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import org.apache.commons.lang3.ObjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.unit.DataUnit;
import java.time.*;
import java.time.temporal.TemporalAccessor;
import java.util.*;
@Service
@ -167,4 +164,73 @@ public class CloudRecordServiceImpl implements ICloudRecordService {
}
return result.getJSONArray("data");
}
@Override
public void changeCollect(String type, boolean result, String app, String stream, String mediaServerId, String startTime, String endTime, String callId, String collectType) {
// 开始时间和结束时间在数据库中都是以秒为单位的
Long startTimeStamp = null;
Long endTimeStamp = null;
if (startTime != null ) {
if (!DateUtil.verification(startTime, DateUtil.formatter)) {
throw new ControllerException(ErrorCode.ERROR100.getCode(), "开始时间格式错误,正确格式为: " + DateUtil.formatter);
}
startTimeStamp = DateUtil.yyyy_MM_dd_HH_mm_ssToTimestamp(startTime);
}
if (endTime != null ) {
if (!DateUtil.verification(endTime, DateUtil.formatter)) {
throw new ControllerException(ErrorCode.ERROR100.getCode(), "结束时间格式错误,正确格式为: " + DateUtil.formatter);
}
endTimeStamp = DateUtil.yyyy_MM_dd_HH_mm_ssToTimestamp(endTime);
}
List<MediaServerItem> mediaServerItems;
if (!ObjectUtils.isEmpty(mediaServerId)) {
mediaServerItems = new ArrayList<>();
MediaServerItem mediaServerItem = mediaServerService.getOne(mediaServerId);
if (mediaServerItem == null) {
throw new ControllerException(ErrorCode.ERROR100.getCode(), "未找到流媒体: " + mediaServerId);
}
mediaServerItems.add(mediaServerItem);
} else {
mediaServerItems = null;
}
List<CloudRecordItem> all = cloudRecordServiceMapper.getList(null, app, stream, startTimeStamp, endTimeStamp,
callId, mediaServerItems);
if (all.isEmpty()) {
throw new ControllerException(ErrorCode.ERROR100.getCode(), "未找到待收藏的视频");
}
int limitCount = 50;
if (all.size() > limitCount) {
for (int i = 0; i < all.size(); i += limitCount) {
int toIndex = i + limitCount;
if (i + limitCount > all.size()) {
toIndex = all.size();
}
if ("collect".equals(collectType)) {
cloudRecordServiceMapper.updateCollectList(result, all.subList(i, toIndex));
}else if ("reserve".equals(collectType)) {
cloudRecordServiceMapper.updateReserveList(result, all.subList(i, toIndex));
}
}
}else {
if ("collect".equals(collectType)) {
cloudRecordServiceMapper.updateCollectList(result, all);
}else if ("reserve".equals(collectType)) {
cloudRecordServiceMapper.updateReserveList(result, all);
}
}
}
@Override
public void changeCollectById(Integer recordId, String collectType, boolean result) {
if ("collect".equals(collectType)) {
cloudRecordServiceMapper.changeCollectById(result, recordId);
}else if ("reserve".equals(collectType)) {
cloudRecordServiceMapper.changeReserveById(result, recordId);
}
}
}

View File

@ -752,4 +752,9 @@ public class MediaServerServiceImpl implements IMediaServerService {
result.setGbSend(redisCatchStorage.getGbSendCount(mediaServerItem.getId()));
return result;
}
@Override
public List<MediaServerItem> getAllWithAssistPort() {
return mediaServerMapper.queryAllWithAssistPort();
}
}

View File

@ -2,10 +2,7 @@ package com.genersoft.iot.vmp.storager.dao;
import com.genersoft.iot.vmp.media.zlm.dto.MediaServerItem;
import com.genersoft.iot.vmp.service.bean.CloudRecordItem;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.*;
import java.util.List;
@ -78,4 +75,42 @@ public interface CloudRecordServiceMapper {
@Param("startTimeStamp")Long startTimeStamp, @Param("endTimeStamp")Long endTimeStamp,
@Param("callId")String callId, List<MediaServerItem> mediaServerItemList);
@Update(" <script>" +
"update wvp_cloud_record set collect = #{collect} where file_path in " +
" <foreach collection='cloudRecordItemList' item='item' open='(' separator=',' close=')' > #{item.filePath}</foreach>" +
" </script>")
void updateCollectList(@Param("collect") boolean collect, List<CloudRecordItem> cloudRecordItemList);
@Delete(" <script>" +
"delete from wvp_cloud_record where media_server_id=#{mediaServerId} file_path in " +
" <foreach collection='filePathList' item='item' open='(' separator=',' close=')' > #{item}</foreach>" +
" </script>")
void deleteByFileList(List<String> filePathList, @Param("mediaServerId") String mediaServerId);
@Select(" <script>" +
"select file_path" +
" from wvp_cloud_record " +
" where collect = false and reserve = false " +
" <if test= 'endTimeStamp != null '> and start_time &lt;= #{endTimeStamp}</if>" +
" <if test= 'callId != null '> and call_id = #{callId}</if>" +
" <if test= 'mediaServerId != null ' > and media_server_id = #{mediaServerId} </if>" +
" </script>")
List<String> queryRecordFilePathListForDelete(@Param("endTimeStamp")Long endTimeStamp, String mediaServerId);
@Update(" <script>" +
"update wvp_cloud_record set reserve = #{reserve} where file_path in " +
" <foreach collection='cloudRecordItems' item='item' open='(' separator=',' close=')' > #{item.filePath}</foreach>" +
" </script>")
void updateReserveList(@Param("reserve") boolean reserve,List<CloudRecordItem> cloudRecordItems);
@Update(" <script>" +
"update wvp_cloud_record set collect = #{collect} where id = #{recordId} " +
" </script>")
void changeCollectById(@Param("collect") boolean collect, @Param("recordId") Integer recordId);
@Update(" <script>" +
"update wvp_cloud_record set reserve = #{reserve} where id = #{recordId} " +
" </script>")
void changeReserveById(@Param("reserve") boolean reserve, Integer recordId);
}

View File

@ -130,4 +130,8 @@ public interface MediaServerMapper {
@Select("SELECT * FROM wvp_media_server WHERE default_server=true")
MediaServerItem queryDefault();
@Select("SELECT * FROM wvp_media_server WHERE record_assist_port > 0")
List<MediaServerItem> queryAllWithAssistPort();
}

View File

@ -1,19 +1,16 @@
package com.genersoft.iot.vmp.vmanager.cloudRecord;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.genersoft.iot.vmp.conf.DynamicTask;
import com.genersoft.iot.vmp.conf.UserSetting;
import com.genersoft.iot.vmp.conf.exception.ControllerException;
import com.genersoft.iot.vmp.media.zlm.SendRtpPortManager;
import com.genersoft.iot.vmp.media.zlm.ZLMServerFactory;
import com.genersoft.iot.vmp.media.zlm.ZlmHttpHookSubscribe;
import com.genersoft.iot.vmp.media.zlm.dto.MediaServerItem;
import com.genersoft.iot.vmp.service.ICloudRecordService;
import com.genersoft.iot.vmp.service.IMediaServerService;
import com.genersoft.iot.vmp.service.bean.CloudRecordItem;
import com.genersoft.iot.vmp.vmanager.bean.ErrorCode;
import com.genersoft.iot.vmp.vmanager.bean.RecordFile;
import com.github.pagehelper.PageInfo;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
@ -36,11 +33,6 @@ import java.util.List;
@RequestMapping("/api/cloud/record")
public class CloudRecordController {
@Autowired
private ZLMServerFactory zlmServerFactory;
@Autowired
private SendRtpPortManager sendRtpPortManager;
private final static Logger logger = LoggerFactory.getLogger(CloudRecordController.class);
@ -50,14 +42,6 @@ public class CloudRecordController {
@Autowired
private IMediaServerService mediaServerService;
@Autowired
private UserSetting userSetting;
@Autowired
private DynamicTask dynamicTask;
@Autowired
private RedisTemplate<Object, Object> redisTemplate;
@ResponseBody
@GetMapping("/date/list")
@ -68,8 +52,8 @@ public class CloudRecordController {
@Parameter(name = "month", description = "月,置空则查询当月", required = false)
@Parameter(name = "mediaServerId", description = "流媒体ID置空则查询全部", required = false)
public List<String> openRtpServer(
@RequestParam String app,
@RequestParam String stream,
@RequestParam(required = true) String app,
@RequestParam(required = true) String stream,
@RequestParam(required = false) int year,
@RequestParam(required = false) int month,
@RequestParam(required = false) String mediaServerId
@ -108,8 +92,8 @@ public class CloudRecordController {
@Parameter(name = "query", description = "检索内容", required = false)
@Parameter(name = "app", description = "应用名", required = false)
@Parameter(name = "stream", description = "流ID", required = false)
@Parameter(name = "page", description = "当前页", required = false)
@Parameter(name = "count", description = "每页查询数量", required = false)
@Parameter(name = "page", description = "当前页", required = true)
@Parameter(name = "count", description = "每页查询数量", required = true)
@Parameter(name = "startTime", description = "开始时间(yyyy-MM-dd HH:mm:ss)", required = false)
@Parameter(name = "endTime", description = "结束时间(yyyy-MM-dd HH:mm:ss)", required = false)
@Parameter(name = "mediaServerId", description = "流媒体ID置空则查询全部流媒体", required = false)
@ -162,16 +146,16 @@ public class CloudRecordController {
@ResponseBody
@GetMapping("/task/add")
@Operation(summary = "添加合并任务")
@Parameter(name = "app", description = "应用名", required = true)
@Parameter(name = "stream", description = "流ID", required = true)
@Parameter(name = "app", description = "应用名", required = false)
@Parameter(name = "stream", description = "流ID", required = false)
@Parameter(name = "mediaServerId", description = "流媒体ID", required = false)
@Parameter(name = "startTime", description = "鉴权ID", required = false)
@Parameter(name = "endTime", description = "鉴权ID", required = false)
@Parameter(name = "callId", description = "鉴权ID", required = false)
@Parameter(name = "remoteHost", description = "返回地址时的远程地址", required = false)
public String addTask(
@RequestParam(required = true) String app,
@RequestParam(required = true) String stream,
@RequestParam(required = false) String app,
@RequestParam(required = false) String stream,
@RequestParam(required = false) String mediaServerId,
@RequestParam(required = false) String startTime,
@RequestParam(required = false) String endTime,
@ -198,19 +182,61 @@ public class CloudRecordController {
@ResponseBody
@GetMapping("/collect/add")
@Operation(summary = "添加收藏")
@Parameter(name = "app", description = "应用名", required = true)
@Parameter(name = "stream", description = "流ID", required = true)
@Parameter(name = "app", description = "应用名", required = false)
@Parameter(name = "stream", description = "流ID", required = false)
@Parameter(name = "mediaServerId", description = "流媒体ID", required = false)
@Parameter(name = "startTime", description = "鉴权ID", required = false)
@Parameter(name = "endTime", description = "鉴权ID", required = false)
@Parameter(name = "callId", description = "鉴权ID", required = false)
@Parameter(name = "collectType", description = "收藏类型", required = false)
public JSONArray addCollect(
@RequestParam(required = false) String taskId,
@Parameter(name = "collectType", description = "收藏类型, collect/reserve", required = false)
public void addCollect(
@RequestParam(required = false) String app,
@RequestParam(required = false) String stream,
@RequestParam(required = false) String mediaServerId,
@RequestParam(required = false) Boolean isEnd
@RequestParam(required = false) String startTime,
@RequestParam(required = false) String endTime,
@RequestParam(required = false) String callId,
@RequestParam(required = false) String collectType,
@RequestParam(required = false) Integer recordId
){
return cloudRecordService.queryTask(taskId, mediaServerId, isEnd);
if (!"collect".equals(collectType) && !"reserve".equals(collectType)) {
collectType = "collect";
}
if (recordId != null) {
cloudRecordService.changeCollectById(recordId, collectType, true);
}else {
cloudRecordService.changeCollect(collectType, true, app, stream, mediaServerId, startTime, endTime, callId, collectType);
}
}
@ResponseBody
@GetMapping("/collect/delete")
@Operation(summary = "移除收藏")
@Parameter(name = "app", description = "应用名", required = false)
@Parameter(name = "stream", description = "流ID", required = false)
@Parameter(name = "mediaServerId", description = "流媒体ID", required = false)
@Parameter(name = "startTime", description = "鉴权ID", required = false)
@Parameter(name = "endTime", description = "鉴权ID", required = false)
@Parameter(name = "callId", description = "鉴权ID", required = false)
@Parameter(name = "collectType", description = "收藏类型, collect/reserve", required = false)
public void deleteCollect(
@RequestParam(required = false) String app,
@RequestParam(required = false) String stream,
@RequestParam(required = false) String mediaServerId,
@RequestParam(required = false) String startTime,
@RequestParam(required = false) String endTime,
@RequestParam(required = false) String callId,
@RequestParam(required = false) String collectType,
@RequestParam(required = false) Integer recordId
){
if (!"collect".equals(collectType) && !"reserve".equals(collectType)) {
collectType = "collect";
}
if (recordId != null) {
cloudRecordService.changeCollectById(recordId, collectType, false);
}else {
cloudRecordService.changeCollect(collectType, false, app, stream, mediaServerId, startTime, endTime, callId, collectType);
}
}